bpo-10544: Disallow "yield" in comprehensions and generator expressions. (GH-4564)

Author: serhiy-storchaka

Doc/reference/expressions.rst

@@ -196,8 +196,7 @@ they may depend on the values obtained from the leftmost iterable. For example:
 
 To ensure the comprehension always results in a container of the appropriate
 type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly
-nested scope (in Python 3.7, such expressions emit :exc:`DeprecationWarning`
-when compiled, in Python 3.8+ they will emit :exc:`SyntaxError`).
+nested scope.
 
 Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for`
 clause may be used to iterate over a :term:`asynchronous iterator`.
@@ -214,8 +213,8 @@ See also :pep:`530`.
 .. versionadded:: 3.6
  Asynchronous comprehensions were introduced.
 
-.. deprecated:: 3.7
- ``yield`` and ``yield from`` deprecated in the implicitly nested scope.
+.. versionchanged:: 3.8
+ ``yield`` and ``yield from`` prohibited in the implicitly nested scope.
 
 
 .. _lists:
@@ -350,9 +349,7 @@ The parentheses can be omitted on calls with only one argument. See section
 
 To avoid interfering with the expected operation of the generator expression
 itself, ``yield`` and ``yield from`` expressions are prohibited in the
-implicitly defined generator (in Python 3.7, such expressions emit
-:exc:`DeprecationWarning` when compiled, in Python 3.8+ they will emit
-:exc:`SyntaxError`).
+implicitly defined generator.
 
 If a generator expression contains either :keyword:`async for`
 clauses or :keyword:`await` expressions it is called an
@@ -368,8 +365,8 @@ which is an asynchronous iterator (see :ref:`async-iterators`).
  only appear in :keyword:`async def` coroutines. Starting
  with 3.7, any function can use asynchronous generator expressions.
 
-.. deprecated:: 3.7
- ``yield`` and ``yield from`` deprecated in the implicitly nested scope.
+.. versionchanged:: 3.8
+ ``yield`` and ``yield from`` prohibited in the implicitly nested scope.
 
 
 .. _yieldexpr:
@@ -401,12 +398,10 @@ coroutine function to be an asynchronous generator. For example::
 
 Due to their side effects on the containing scope, ``yield`` expressions
 are not permitted as part of the implicitly defined scopes used to
-implement comprehensions and generator expressions (in Python 3.7, such
-expressions emit :exc:`DeprecationWarning` when compiled, in Python 3.8+
-they will emit :exc:`SyntaxError`)..
+implement comprehensions and generator expressions.
 
-.. deprecated:: 3.7
- Yield expressions deprecated in the implicitly nested scopes used to
+.. versionchanged:: 3.8
+ Yield expressions prohibited in the implicitly nested scopes used to
  implement comprehensions and generator expressions.
 
 Generator functions are described below, while asynchronous generator

Doc/whatsnew/3.8.rst

@@ -113,6 +113,15 @@ This section lists previously described changes and other bugfixes
 that may require changes to your code.
 
 
+Changes in Python behavior
+--------------------------
+
+* Yield expressions (both ``yield`` and ``yield from`` clauses) are now disallowed
+ in comprehensions and generator expressions (aside from the iterable expression
+ in the leftmost :keyword:`for` clause).
+ (Contributed by Serhiy Storchaka in :issue:`10544`.)
+
+
 Changes in the Python API
 -------------------------
 

Lib/test/support/__init__.py

@@ -1061,8 +1061,8 @@ def make_bad_fd():
 file.close()
 unlink(TESTFN)
 
-defcheck_syntax_error(testcase, statement, *, lineno=None, offset=None):
-withtestcase.assertRaises(SyntaxError) ascm:
+defcheck_syntax_error(testcase, statement, errtext='', *, lineno=None, offset=None):
+withtestcase.assertRaisesRegex(SyntaxError, errtext) ascm:
 compile(statement, '<test string>', 'exec')
 err=cm.exception
 testcase.assertIsNotNone(err.lineno)

Lib/test/test_grammar.py

@@ -251,6 +251,8 @@ def __getitem__(self, item):
 
 classGrammarTests(unittest.TestCase):
 
+check_syntax_error=check_syntax_error
+
 # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
 # XXX can't test in a script -- this rule is only used when interactive
 
@@ -920,15 +922,7 @@ def test_yield_in_comprehensions(self):
 defg(): [xforxin [(yield1)]]
 defg(): [xforxin [(yieldfrom ())]]
 
-defcheck(code, warntext):
-withself.assertWarnsRegex(DeprecationWarning, warntext):
-compile(code, '<test string>', 'exec')
-importwarnings
-withwarnings.catch_warnings():
-warnings.filterwarnings('error', category=DeprecationWarning)
-withself.assertRaisesRegex(SyntaxError, warntext):
-compile(code, '<test string>', 'exec')
-
+check=self.check_syntax_error
 check("def g(): [(yield x) for x in ()]",
 "'yield' inside list comprehension")
 check("def g(): [x for x in () if not (yield x)]",

Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst

@@ -0,0 +1,2 @@
+Yield expressions are now disallowed in comprehensions and generator
+expressions except the expression for the outermost iterable.

Python/symtable.c

@@ -1754,35 +1754,18 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
 VISIT(st, expr, value);
 VISIT(st, expr, elt);
 if (st->st_cur->ste_generator) {
-PyObject*msg=PyUnicode_FromString(
+PyErr_SetString(PyExc_SyntaxError,
  (e->kind==ListComp_kind) ? "'yield' inside list comprehension" :
  (e->kind==SetComp_kind) ? "'yield' inside set comprehension" :
  (e->kind==DictComp_kind) ? "'yield' inside dict comprehension" :
 "'yield' inside generator expression");
-if (msg==NULL) {
-symtable_exit_block(st, (void*)e);
-return0;
- }
-if (PyErr_WarnExplicitObject(PyExc_DeprecationWarning,
-msg, st->st_filename, st->st_cur->ste_lineno,
-NULL, NULL) ==-1)
- {
-if (PyErr_ExceptionMatches(PyExc_DeprecationWarning)) {
-/* Replace the DeprecationWarning exception with a SyntaxError
- to get a more accurate error report */
-PyErr_Clear();
-PyErr_SetObject(PyExc_SyntaxError, msg);
-PyErr_SyntaxLocationObject(st->st_filename,
-st->st_cur->ste_lineno,
-st->st_cur->ste_col_offset);
- }
-Py_DECREF(msg);
-symtable_exit_block(st, (void*)e);
-return0;
- }
-Py_DECREF(msg);
+PyErr_SyntaxLocationObject(st->st_filename,
+st->st_cur->ste_lineno,
+st->st_cur->ste_col_offset);
+symtable_exit_block(st, (void*)e);
+return0;
  }
-st->st_cur->ste_generator|= is_generator;
+st->st_cur->ste_generator=is_generator;
 returnsymtable_exit_block(st, (void*)e);
 }