Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 53 additions & 4 deletions mypyc/irbuild/classdef.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from typing import Final

from mypy.nodes import (
ARG_POS,
EXCLUDED_ENUM_ATTRIBUTES,
TYPE_VAR_TUPLE_KIND,
AssignmentStmt,
Expand Down Expand Up @@ -745,6 +746,20 @@ def find_attr_initializers(
) -> tuple[set[str], list[tuple[AssignmentStmt, str]]]:
"""Find initializers of attributes in a class body.

Under separate compilation, only this class's own body is walked, and
generate_attr_defaults_init emits a runtime call to the parent's
__mypyc_defaults_setup so inherited defaults are produced by chaining,
not by inlining. Walking the MRO here would break under separate=True
with mypy's incremental cache: a base class loaded from the cache has
an empty ClassDef.defs.body (mypy/nodes.py::ClassDef.serialize doesn't
serialize the class body), so inherited assignments would be silently
dropped and the subclass's __mypyc_defaults_setup would leave inherited
slots in the "undefined" state at runtime.

Without separate compilation, all modules are parsed in the same pass
and the MRO walk is safe; we keep the original inline-all behavior
there as an optimization (no chain call needed for instance creation).

If provided, the skip arg should be a callable which will return whether
to skip generating a default for an attribute. It will be passed the name of
the attribute and the corresponding AssignmentStmt.
Expand All @@ -758,7 +773,12 @@ def find_attr_initializers(
# Pull out all assignments in classes in the mro so we can initialize them
# TODO: Support nested statements
default_assignments: list[tuple[AssignmentStmt, str]] = []
for info in reversed(cdef.info.mro):
if builder.options.separate:
infos: list[TypeInfo] = [cdef.info]
else:
infos = list(reversed(cdef.info.mro))

for info in infos:
if info not in builder.mapper.type_to_ir:
continue
for stmt in info.defn.defs.body:
Expand Down Expand Up @@ -800,15 +820,44 @@ def find_attr_initializers(
def generate_attr_defaults_init(
builder: IRBuilder, cdef: ClassDef, default_assignments: list[tuple[AssignmentStmt, str]]
) -> None:
"""Generate an initialization method for default attr values (from class vars)."""
if not default_assignments:
return
"""Generate an initialization method for default attr values (from class vars).

Under separate compilation, the emitted __mypyc_defaults_setup chains to
the nearest ancestor that has the method (Python __init__ style), then
sets only this class's own defaults; inherited defaults are produced by
the chain at runtime. Without separate compilation, find_attr_initializers
has already collected the full MRO's defaults into default_assignments,
so we inline them all as before.
"""
cls = builder.mapper.type_to_ir[cdef.info]
if cls.builtin_base:
return

parent_with_defaults: ClassIR | None = None
if builder.options.separate:
for ancestor in cls.mro[1:]:
if "__mypyc_defaults_setup" in ancestor.method_decls:
parent_with_defaults = ancestor
break
Comment on lines +839 to +841
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this makes the compilation results dependent on the order in which files are compiled, because __mypyc_defaults_setup will only be present in method_decls after it has been added when this function is called for the base class. so if the subclass is compiled before the base class then it won't find it. assuming the base and subclasses are defined in different files like in the test.

in the test this is not hit because the three different files are each in their own compilation groups, and the dependency analysis finds that they are dependent on each other so the compilation order is always the same: other_b.py (base class) -> other_a.py (subclass) -> native.py (instantiation).

but if we put other_b.py and other_a.py in the same compilation group with this comment:

[case testIncrementalCrossModuleInheritedAttrDefaultsWithOverride]
# separate: [(["native.py"], "grp1"), (["other_a.py", "other_b.py"], "grp2")]
...

then the compilation order changes because the dependency analysis treats all files in a given group as dependent on each other, and the compilation order within a group is determined by filenames. so other_a.py (subclass) will be compiled before other_b.py (base class) because its name comes first when sorting.

to make the compilation independent on the order, we have a preparation phase in prepare.py that runs before the IR build and sets up attributes that might be needed when compiling across different files.

so i think we should move the analysis of whether __mypyc_defaults_setup needs to be generated to prepare.py, and add the method declaration there if so. then here we only generate the body and we can safely make this check.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, this is very helpful, appreciate the detailed explanation and it makes sense. Let me give your suggestion a whirl. I'll make sure to include a regression test covering this path as well.


if not default_assignments and parent_with_defaults is None:
return

with builder.enter_method(cls, "__mypyc_defaults_setup", bool_rprimitive):
self_var = builder.self()

# Chain to parent's setup so inherited defaults run first; propagate
# its False return so a parent default that raised still aborts
# instance creation rather than being silently swallowed here.
if parent_with_defaults is not None:
decl = parent_with_defaults.method_decl("__mypyc_defaults_setup")
parent_ok = builder.builder.call(decl, [self_var], [ARG_POS], [None], cdef.line)
fail_block, continue_block = BasicBlock(), BasicBlock()
builder.add(Branch(parent_ok, continue_block, fail_block, Branch.BOOL))
builder.activate_block(fail_block)
builder.add(Return(builder.false()))
builder.activate_block(continue_block)

for stmt, origin_module in default_assignments:
lvalue = stmt.lvalues[0]
assert isinstance(lvalue, NameExpr), lvalue
Expand Down
53 changes: 53 additions & 0 deletions mypyc/test-data/run-multimodule.test
Original file line number Diff line number Diff line change
Expand Up @@ -1778,3 +1778,56 @@ hello
[out2]
empty
hello

[case testIncrementalCrossModuleInheritedAttrDefaultsWithOverride]
# Regression: same shape as testIncrementalCrossModuleInheritedAttrDefaults,
# but the subclass adds an attribute of its own, so generate_attr_defaults_init
# emits a __mypyc_defaults_setup for it. Before the fix, the recompiled
# subclass walked the parent's ClassDef.defs.body to collect inherited
# defaults; when the parent was loaded from mypy's incremental cache that
# body was empty, so the inherited initialization was dropped and any
# access to an inherited attribute through compiled code raised
# "AttributeError: attribute '<name>' of '<base>' undefined".
import other_a

def test() -> None:
c = other_a.Child()
# Inherited attributes must still be initialized after the subclass
# has been recompiled against a cache-loaded parent.
assert c.x == 1
assert c.y == "hello"
# Own override is set by the subclass's own __mypyc_defaults_setup.
assert c.z is True
# Method defined on the parent reads an inherited attribute through
# the compiled path; this is what crashes pre-fix.
assert c.use() == 1

[file other_b.py]
class Parent:
x: int = 1
y: str = "hello"
z: bool = False

def use(self) -> int:
if self.x:
return 1
return 0

[file other_a.py]
from other_b import Parent

class Child(Parent):
z: bool = True

[file other_a.py.2]
from other_b import Parent

class Child(Parent):
z: bool = True

def _force_recompile() -> int:
return 1

[file driver.py]
from native import test
test()
Loading