gh-150052: Resolve un-loaded lazily loaded submodules via module.__getattr__ instead of publishing lazy values (#150055) (#150744)

Author: johnslavik

Include/internal/pycore_import.h

@@ -39,6 +39,8 @@ extern PyObject * _PyImport_GetAbsName(
 // Symbol is exported for the JIT on Windows builds.
 PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate(
     PyThreadState *tstate, PyObject *lazy_import);
+extern PyObject * _PyImport_TryLoadLazySubmodule(
+    PyObject *mod_name, PyObject *attr_name);
 extern PyObject * _PyImport_LazyImportModuleLevelObject(
     PyThreadState *tstate, PyObject *name, PyObject *builtins,
     PyObject *globals, PyObject *locals, PyObject *fromlist, int level);

Lib/test/test_lazy_import/__init__.py

@@ -503,6 +503,14 @@ def test_lazy_import_pkg(self):
         self.assertIn("test.test_lazy_import.data.pkg.bar", sys.modules)
         self.assertIn("BAR_MODULE_LOADED", out.getvalue())
 
+    def test_lazy_submodule_stored_in_parent_dict(self):
+        """Accessing a lazy submodule should store it in the parent's __dict__."""
+        import test.test_lazy_import.data.lazy_import_pkg
+
+        pkg = sys.modules["test.test_lazy_import.data.pkg"]
+        self.assertIn("bar", pkg.__dict__)
+        self.assertIs(pkg.__dict__["bar"], sys.modules["test.test_lazy_import.data.pkg.bar"])
+
     def test_lazy_import_pkg_cross_import(self):
         """Cross-imports within package should preserve lazy imports."""
         import test.test_lazy_import.data.pkg.c
@@ -515,6 +523,18 @@ def test_lazy_import_pkg_cross_import(self):
         self.assertEqual(type(g["x"]), int)
         self.assertEqual(type(g["b"]), types.LazyImportType)
 
+    @support.requires_subprocess()
+    def test_lazy_from_import_does_not_pollute_parent(self):
+        """Lazy from import should not add the name to the parent module's dict."""
+        code = textwrap.dedent("""
+            lazy from json import nonexistent_attr
+            import json
+            assert "nonexistent_attr" not in json.__dict__, (
+                "lazy from import should not publish attributes on the parent module"
+            )
+        """)
+        assert_python_ok("-c", code)
+
     @support.requires_subprocess()
     def test_package_from_import_with_module_getattr_raising(self):
         """Lazy from import should respect a package's __getattr__."""
@@ -716,19 +736,14 @@ def tearDown(self):
         sys.set_lazy_imports("normal")
 
     def test_import_error_shows_chained_traceback(self):
-        """ImportError during reification should chain to show both definition and access."""
-        # Errors at reification must show where the lazy import was defined
-        # AND where the access happened, per PEP 810 "Reification" section
+        """Accessing a nonexistent lazy submodule via parent attr raises AttributeError."""
         code = textwrap.dedent("""
             import sys
             lazy import test.test_lazy_import.data.nonexistent_module
 
             try:
                 x = test.test_lazy_import.data.nonexistent_module
-            except ImportError as e:
-                # Should have __cause__ showing the original error
-                # The exception chain shows both where import was defined and where access happened
-                assert e.__cause__ is not None, "Expected chained exception"
+            except AttributeError as e:
                 print("OK")
         """)
         result = subprocess.run(
@@ -776,7 +791,7 @@ def test_reification_retries_on_failure(self):
             # First access - should fail
             try:
                 x = test.test_lazy_import.data.broken_module
-            except ValueError:
+            except AttributeError:
                 pass
 
             # The lazy object should still be a lazy proxy (not reified)
@@ -786,7 +801,7 @@ def test_reification_retries_on_failure(self):
             # Second access - should also fail (retry the import)
             try:
                 x = test.test_lazy_import.data.broken_module
-            except ValueError:
+            except AttributeError:
                 print("OK - retry worked")
         """)
         result = subprocess.run(
@@ -799,20 +814,15 @@ def test_reification_retries_on_failure(self):
 
     def test_error_during_module_execution_propagates(self):
         """Errors in module code during reification should propagate correctly."""
-        # Module that raises during import should propagate with chaining
         code = textwrap.dedent("""
             import sys
             lazy import test.test_lazy_import.data.broken_module
 
             try:
                 _ = test.test_lazy_import.data.broken_module
                 print("FAIL - should have raised")
-            except ValueError as e:
-                # The ValueError from the module should be the cause
-                if "always fails" in str(e) or (e.__cause__ and "always fails" in str(e.__cause__)):
-                    print("OK")
-                else:
-                    print(f"FAIL - wrong error: {e}")
+            except AttributeError:
+                print("OK")
         """)
         result = subprocess.run(
             [sys.executable, "-c", code],

Objects/moduleobject.c

@@ -1299,6 +1299,33 @@ _PyModule_IsPossiblyShadowing(PyObject *origin)
     return result;
 }
 
+// Check if `name` is a lazily pending submodule of module `m`.
+// Returns a new reference on success, or NULL with no error set.
+static PyObject *
+try_load_lazy_submodule(PyModuleObject *m, PyObject *name)
+{
+    PyObject *mod_name;
+    int rc = PyDict_GetItemRef(m->md_dict, &_Py_ID(__name__), &mod_name);
+    if (rc <= 0) {
+        return NULL;
+    }
+    if (!PyUnicode_Check(mod_name)) {
+        Py_DECREF(mod_name);
+        return NULL;
+    }
+    PyObject *result = _PyImport_TryLoadLazySubmodule(mod_name, name);
+    Py_DECREF(mod_name);
+    if (result == NULL) {
+        PyErr_Clear();
+        return NULL;
+    }
+    if (PyDict_SetItem(m->md_dict, name, result) < 0) {
+        Py_DECREF(result);
+        return NULL;
+    }
+    return result;
+}
+
 PyObject*
 _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress)
 {
@@ -1363,6 +1390,13 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress)
         PyErr_Clear();
     }
     assert(m->md_dict != NULL);
+    attr = try_load_lazy_submodule(m, name);
+    if (attr != NULL) {
+        return attr;
+    }
+    if (PyErr_Occurred()) {
+        return NULL;
+    }
     if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) {
         return NULL;
     }