Skip to content

GH-143493: Conform to spec for generator expressions while supporting virtual iterators #143569

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
markshannon wants to merge 2 commits into
base: main
Choose a base branch
from
Open

GH-143493: Conform to spec for generator expressions while supporting virtual iterators #143569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4deca76
Conform to spec for generator expressions while supporting virtual it…
markshannon Jan 6, 2026
0d9ded3
Add news
markshannon Jan 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Include/internal/pycore_flowgraph.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ int _PyCfg_OptimizeCodeUnit(struct _PyCfgBuilder *g, PyObject *consts, PyObject
struct _PyCfgBuilder* _PyCfg_FromInstructionSequence(_PyInstructionSequence *seq);
int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyInstructionSequence *seq);
int _PyCfg_OptimizedCfgToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_CodeUnitMetadata *umd,
int code_flags, int *stackdepth, int *nlocalsplus,
int *stackdepth, int *nlocalsplus,
_PyInstructionSequence *seq);

PyCodeObject *
Expand Down
1 change: 1 addition & 0 deletions Include/internal/pycore_instruction_sequence.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ int _PyInstructionSequence_SetAnnotationsCode(_PyInstructionSequence *seq,
_PyInstructionSequence *annotations);
int _PyInstructionSequence_AddNested(_PyInstructionSequence *seq, _PyInstructionSequence *nested);
void PyInstructionSequence_Fini(_PyInstructionSequence *seq);
_PyInstruction _PyInstructionSequence_GetInstruction(_PyInstructionSequence *seq, int pos);

extern PyTypeObject _PyInstructionSequence_Type;
#define _PyInstructionSequence_Check(v) Py_IS_TYPE((v), &_PyInstructionSequence_Type)
Expand Down
3 changes: 2 additions & 1 deletion Include/internal/pycore_magic_number.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,7 @@ Known values:
Python 3.15a4 3657 (Add BINARY_OP_SUBSCR_USTR_INT)
Python 3.15a4 3658 (Optimize bytecode for list/set called on genexp)
Python 3.15a4 3659 (Add CALL_FUNCTION_EX specialization)
Python 3.15a4 3660 (Change generator preamble code)


Python 3.16 will start with 3700
Expand All @@ -303,7 +304,7 @@ PC/launcher.c must also be updated.

*/

#define PYC_MAGIC_NUMBER 3659
#define PYC_MAGIC_NUMBER 3660
/* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
(little-endian) and then appending b'\r\n'. */
#define PYC_MAGIC_NUMBER_TOKEN \
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_compile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1298,7 +1298,7 @@ def return_genexp():
x
in
y)
genexp_lines = [0, 4, 2, 0, 4]
genexp_lines = [4, 0, 4, 2, 0, 4]

genexp_code = return_genexp.__code__.co_consts[0]
code_lines = self.get_code_lines(genexp_code)
Expand Down
8 changes: 4 additions & 4 deletions Lib/test/test_dis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -875,11 +875,11 @@ def foo(x):
Disassembly of <code object <genexpr> at 0x..., file "%s", line %d>:
-- COPY_FREE_VARS 1
%4d RETURN_GENERATOR
%4d LOAD_FAST 0 (.0)
GET_ITER
RETURN_GENERATOR
POP_TOP
L1: RESUME 0
LOAD_FAST 0 (.0)
GET_ITER
L2: FOR_ITER 14 (to L3)
STORE_FAST 1 (z)
LOAD_DEREF 2 (x)
Expand All @@ -897,7 +897,7 @@ def foo(x):
-- L4: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR)
RERAISE 1
ExceptionTable:
L1 to L4 -> L4 [0] lasti
L1 to L4 -> L4 [2] lasti
""" % (dis_nested_1,
__file__,
_h.__code__.co_firstlineno + 3,
Expand Down
34 changes: 19 additions & 15 deletions Lib/test/test_generators.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,21 +357,25 @@ def gen(it):
yield x
return gen(range(10))

def process_tests(self, get_generator):
for obj in self.iterables:
g_obj = get_generator(obj)
with self.subTest(g_obj=g_obj, obj=obj):
self.assertListEqual(list(g_obj), list(obj))
def process_tests(self, get_generator, changes_iterable):
if changes_iterable:
for obj in self.iterables:
g_obj = get_generator(obj)
with self.subTest(g_obj=g_obj, obj=obj):
self.assertListEqual(list(g_obj), list(obj))

g_iter = get_generator(iter(obj))
with self.subTest(g_iter=g_iter, obj=obj):
self.assertListEqual(list(g_iter), list(obj))
g_iter = get_generator(iter(obj))
with self.subTest(g_iter=g_iter, obj=obj):
self.assertListEqual(list(g_iter), list(obj))

err_regex = "'.*' object is not iterable"
for obj in self.non_iterables:
g_obj = get_generator(obj)
with self.subTest(g_obj=g_obj):
self.assertRaisesRegex(TypeError, err_regex, list, g_obj)
if changes_iterable:
self.assertRaisesRegex(TypeError, err_regex, list, g_obj)
else:
next(g_obj)

def test_modify_f_locals(self):
def modify_f_locals(g, local, obj):
Expand All @@ -384,22 +388,22 @@ def get_generator_genexpr(obj):
def get_generator_genfunc(obj):
return modify_f_locals(self.genfunc(), 'it', obj)

self.process_tests(get_generator_genexpr)
self.process_tests(get_generator_genfunc)
self.process_tests(get_generator_genexpr, False)
self.process_tests(get_generator_genfunc, True)

def test_new_gen_from_gi_code(self):
def new_gen_from_gi_code(g, obj):
generator_func = types.FunctionType(g.gi_code, {})
return generator_func(obj)

def get_generator_genexpr(obj):
return new_gen_from_gi_code(self.genexpr(), obj)
for obj in self.non_iterables:
with self.assertRaises(TypeError):
new_gen_from_gi_code(self.genexpr(), obj)

def get_generator_genfunc(obj):
return new_gen_from_gi_code(self.genfunc(), obj)

self.process_tests(get_generator_genexpr)
self.process_tests(get_generator_genfunc)
self.process_tests(get_generator_genfunc, True)


class ExceptionTest(unittest.TestCase):
Expand Down
3 changes: 3 additions & 0 deletions Misc/NEWS.d/next/Core_and_Builtins/2026-01-08-14-55-31.gh-issue-143569.-Ltu3c.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Generator expressions in 3.15 now conform to the documented behavior when
the iterable does not support iteration. This matches the behavior in 3.14
and earlier
5 changes: 3 additions & 2 deletions Python/bytecodes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3166,14 +3166,15 @@ dummy_func(
#ifdef Py_STATS
_Py_GatherStats_GetIter(iterable);
#endif
/* before: [obj]; after [getiter(obj)] */
PyTypeObject *tp = PyStackRef_TYPE(iterable);
if (tp == &PyTuple_Type || tp == &PyList_Type) {
/* Leave iterable on stack and pushed tagged 0 */
iter = iterable;
DEAD(iterable);
index_or_null = PyStackRef_TagInt(0);
}
else {
/* Pop iterable, and push iterator then NULL */
PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable));
PyStackRef_CLOSE(iterable);
ERROR_IF(iter_o == NULL);
Expand Down Expand Up @@ -5010,7 +5011,7 @@ dummy_func(
PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
ERROR_IF(gen == NULL);
assert(STACK_LEVEL() == 0);
assert(STACK_LEVEL() <= 2);
SAVE_STACK();
_PyInterpreterFrame *gen_frame = &gen->gi_iframe;
frame->instr_ptr++;
Expand Down
76 changes: 61 additions & 15 deletions Python/codegen.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,19 +227,25 @@ static int codegen_call_helper(compiler *c, location loc,
static int codegen_try_except(compiler *, stmt_ty);
static int codegen_try_star_except(compiler *, stmt_ty);

typedef enum {
ITERABLE_IN_LOCAL = 0,
ITERABLE_ON_STACK = 1,
ITERATOR_ON_STACK = 2,
} IterStackState;

static int codegen_sync_comprehension_generator(
compiler *c, location loc,
asdl_comprehension_seq *generators, int gen_index,
int depth,
expr_ty elt, expr_ty val, int type,
int iter_on_stack);
IterStackState iter_on_stack);

static int codegen_async_comprehension_generator(
compiler *c, location loc,
asdl_comprehension_seq *generators, int gen_index,
int depth,
expr_ty elt, expr_ty val, int type,
int iter_on_stack);
IterStackState iter_on_stack);

static int codegen_pattern(compiler *, pattern_ty, pattern_context *);
static int codegen_match(compiler *, stmt_ty);
Expand Down Expand Up @@ -665,6 +671,18 @@ codegen_enter_scope(compiler *c, identifier name, int scope_type,
if (scope_type == COMPILE_SCOPE_MODULE) {
loc.lineno = 0;
}
/* Add the generator prefix instructions. */

PySTEntryObject *ste = SYMTABLE_ENTRY(c);
if (ste->ste_coroutine || ste->ste_generator) {
/* Note that RETURN_GENERATOR + POP_TOP have a net stack effect
* of 0. This is because RETURN_GENERATOR pushes the generator
before returning. */
location loc = LOCATION(lineno, lineno, -1, -1);
ADDOP(c, loc, RETURN_GENERATOR);
ADDOP(c, loc, POP_TOP);
}

ADDOP_I(c, loc, RESUME, RESUME_AT_FUNC_START);
if (scope_type == COMPILE_SCOPE_MODULE) {
ADDOP(c, loc, ANNOTATIONS_PLACEHOLDER);
Expand Down Expand Up @@ -1187,10 +1205,15 @@ codegen_wrap_in_stopiteration_handler(compiler *c)
{
NEW_JUMP_TARGET_LABEL(c, handler);

/* Insert SETUP_CLEANUP at start */
/* Insert SETUP_CLEANUP just before RESUME */
instr_sequence *seq = INSTR_SEQUENCE(c);
int resume = 0;
while (_PyInstructionSequence_GetInstruction(seq, resume).i_opcode != RESUME) {
resume++;
}
RETURN_IF_ERROR(
_PyInstructionSequence_InsertInstruction(
INSTR_SEQUENCE(c), 0,
seq, resume,
SETUP_CLEANUP, handler.id, NO_LOCATION));

ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None);
Expand Down Expand Up @@ -4401,7 +4424,7 @@ codegen_comprehension_generator(compiler *c, location loc,
asdl_comprehension_seq *generators, int gen_index,
int depth,
expr_ty elt, expr_ty val, int type,
int iter_on_stack)
IterStackState iter_on_stack)
{
comprehension_ty gen;
gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
Expand All @@ -4421,7 +4444,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
asdl_comprehension_seq *generators,
int gen_index, int depth,
expr_ty elt, expr_ty val, int type,
int iter_on_stack)
IterStackState iter_on_stack)
{
/* generate code for the iterator, then each of the ifs,
and then write to the element */
Expand All @@ -4433,7 +4456,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators,
gen_index);

if (!iter_on_stack) {
if (iter_on_stack == ITERABLE_IN_LOCAL) {
if (gen_index == 0) {
assert(METADATA(c)->u_argcount == 1);
ADDOP_I(c, loc, LOAD_FAST, 0);
Expand Down Expand Up @@ -4468,9 +4491,12 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
}

if (IS_JUMP_TARGET_LABEL(start)) {
depth += 2;
ADDOP(c, LOC(gen->iter), GET_ITER);
if (iter_on_stack != ITERATOR_ON_STACK) {
ADDOP(c, LOC(gen->iter), GET_ITER);
depth += 1;
}
USE_LABEL(c, start);
depth += 1;
ADDOP_JUMP(c, LOC(gen->iter), FOR_ITER, anchor);
}
VISIT(c, expr, gen->target);
Expand All @@ -4486,7 +4512,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
RETURN_IF_ERROR(
codegen_comprehension_generator(c, loc,
generators, gen_index, depth,
elt, val, type, 0));
elt, val, type, ITERABLE_IN_LOCAL));
}

location elt_loc = LOC(elt);
Expand Down Expand Up @@ -4545,7 +4571,7 @@ codegen_async_comprehension_generator(compiler *c, location loc,
asdl_comprehension_seq *generators,
int gen_index, int depth,
expr_ty elt, expr_ty val, int type,
int iter_on_stack)
IterStackState iter_on_stack)
{
NEW_JUMP_TARGET_LABEL(c, start);
NEW_JUMP_TARGET_LABEL(c, send);
Expand All @@ -4555,7 +4581,7 @@ codegen_async_comprehension_generator(compiler *c, location loc,
comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators,
gen_index);

if (!iter_on_stack) {
if (iter_on_stack == ITERABLE_IN_LOCAL) {
if (gen_index == 0) {
assert(METADATA(c)->u_argcount == 1);
ADDOP_I(c, loc, LOAD_FAST, 0);
Expand All @@ -4565,7 +4591,9 @@ codegen_async_comprehension_generator(compiler *c, location loc,
VISIT(c, expr, gen->iter);
}
}
ADDOP(c, LOC(gen->iter), GET_AITER);
if (iter_on_stack != ITERATOR_ON_STACK) {
ADDOP(c, LOC(gen->iter), GET_AITER);
}

USE_LABEL(c, start);
/* Runtime will push a block here, so we need to account for that */
Expand Down Expand Up @@ -4795,11 +4823,13 @@ codegen_comprehension(compiler *c, expr_ty e, int type,
location loc = LOC(e);

outermost = (comprehension_ty) asdl_seq_GET(generators, 0);
IterStackState iter_state;
if (is_inlined) {
VISIT(c, expr, outermost->iter);
if (push_inlined_comprehension_state(c, loc, entry, &inline_state)) {
goto error;
}
iter_state = ITERABLE_ON_STACK;
}
else {
/* Receive outermost iter as an implicit argument */
Expand All @@ -4810,6 +4840,23 @@ codegen_comprehension(compiler *c, expr_ty e, int type,
(void *)e, e->lineno, NULL, &umd) < 0) {
goto error;
}
if (type == COMP_GENEXP) {
/* Insert GET_ITER before RETURN_GENERATOR.
https://docs.python.org/3/reference/expressions.html#generator-expressions */
RETURN_IF_ERROR(
_PyInstructionSequence_InsertInstruction(
INSTR_SEQUENCE(c), 0,
LOAD_FAST, 0, LOC(outermost->iter)));
RETURN_IF_ERROR(
_PyInstructionSequence_InsertInstruction(
INSTR_SEQUENCE(c), 1,
outermost->is_async ? GET_AITER : GET_ITER,
0, LOC(outermost->iter)));
iter_state = ITERATOR_ON_STACK;
}
else {
iter_state = ITERABLE_IN_LOCAL;
}
}
Py_CLEAR(entry);

Expand All @@ -4836,9 +4883,8 @@ codegen_comprehension(compiler *c, expr_ty e, int type,
ADDOP_I(c, loc, SWAP, 2);
}
}

if (codegen_comprehension_generator(c, loc, generators, 0, 0,
elt, val, type, is_inlined) < 0) {
elt, val, type, iter_state) < 0) {
goto error_in_scope;
}

Expand Down
4 changes: 2 additions & 2 deletions Python/compile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1443,7 +1443,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache,

int stackdepth;
int nlocalsplus;
if (_PyCfg_OptimizedCfgToInstructionSequence(g, &u->u_metadata, code_flags,
if (_PyCfg_OptimizedCfgToInstructionSequence(g, &u->u_metadata,
&stackdepth, &nlocalsplus,
&optimized_instrs) < 0) {
goto error;
Expand Down Expand Up @@ -1718,7 +1718,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename,

int code_flags = 0;
int stackdepth, nlocalsplus;
if (_PyCfg_OptimizedCfgToInstructionSequence(g, umd, code_flags,
if (_PyCfg_OptimizedCfgToInstructionSequence(g, umd,
&stackdepth, &nlocalsplus,
&optimized_instrs) < 0) {
goto error;
Expand Down
3 changes: 1 addition & 2 deletions Python/executor_cases.c.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading