python · ericsnowcurrently · Oct 5, 2021 · Sep 28, 2021 · Sep 28, 2021 · Sep 29, 2021
@@ -10,6 +10,13 @@ extern PyStatus _PyImport_ReInitLock(void);
 #endif
 extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate);
 
+struct _module_alias {
+    const char *name;                 /* ASCII encoded string */
+    const char *orig;                 /* ASCII encoded string */
+};
+
+extern const struct _module_alias * _PyImport_FrozenAliases;
+
 #ifdef __cplusplus
 }
 #endif

@@ -824,16 +824,39 @@ def module_repr(m):
                        "slated for removal in Python 3.12", DeprecationWarning)
         return '<module {!r} ({})>'.format(m.__name__, FrozenImporter._ORIGIN)
 
+    @classmethod
+    def _setup_module(cls, module):
+        assert not hasattr(module, '__file__'), module.__file__
+        ispkg = hasattr(module, '__path__')
+        assert not ispkg or not module.__path__, module.__path__
+        spec = module.__spec__
+        assert not ispkg or not spec.submodule_search_locations
+
+        if spec.loader_state is None:
+            spec.loader_state = type(sys.implementation)(
+                data=None,
+                origname=None,
+            )
+        elif not hasattr(spec.loader_state, 'data'):
+            spec.loader_state.data = None
+        if not getattr(spec.loader_state, 'origname', None):
+            origname = vars(module).pop('__origname__', None)
+            assert origname, 'see PyImport_ImportFrozenModuleObject()'
+            spec.loader_state.origname = origname
+
     @classmethod
     def find_spec(cls, fullname, path=None, target=None):
         info = _call_with_frames_removed(_imp.find_frozen, fullname)
         if info is None:
             return None
-        data, ispkg = info
+        data, ispkg, origname = info
         spec = spec_from_loader(fullname, cls,
                                 origin=cls._ORIGIN,
                                 is_package=ispkg)
-        spec.loader_state = data
+        spec.loader_state = type(sys.implementation)(
+            data=data,
+            origname=origname,
+        )
         return spec
 
     @classmethod
@@ -857,7 +880,7 @@ def exec_module(module):
         spec = module.__spec__
         name = spec.name
         try:
-            data = spec.loader_state
+            data = spec.loader_state.data
         except AttributeError:
             if not _imp.is_frozen(name):
                 raise ImportError('{!r} is not a frozen module'.format(name),
@@ -868,7 +891,7 @@ def exec_module(module):
             # Note that if this method is called again (e.g. by
             # importlib.reload()) then _imp.get_frozen_object() will notice
             # no data was provided and will look it up.
-            spec.loader_state = None
+            spec.loader_state.data = None
         code = _call_with_frames_removed(_imp.get_frozen_object, name, data)
         exec(code, module.__dict__)
 
@@ -1220,6 +1243,8 @@ def _setup(sys_module, _imp_module):
                 continue
             spec = _spec_from_module(module, loader)
             _init_module_attrs(spec, module)
+            if loader is FrozenImporter:
+                loader._setup_module(module)
 
     # Directly load built-in modules needed during bootstrap.
     self_module = sys.modules[__name__]

@@ -9,7 +9,15 @@
 import unittest
 import warnings
 
-from test.support import import_helper, REPO_ROOT
+from test.support import import_helper, REPO_ROOT, STDLIB_DIR
+
+
+def resolve_stdlib_file(name, ispkg=False):
+    assert name
+    if ispkg:
+        return os.path.join(STDLIB_DIR, *name.split('.'), '__init__.py')
+    else:
+        return os.path.join(STDLIB_DIR, *name.split('.')) + '.py'
 
 
 class FindSpecTests(abc.FinderTests):
@@ -32,16 +40,30 @@ def check_basic(self, spec, name, ispkg=False):
             self.assertIsNone(spec.submodule_search_locations)
         self.assertIsNotNone(spec.loader_state)
 
-    def check_data(self, spec):
+    def check_loader_state(self, spec, origname=None, filename=None):
+        if not filename:
+            if not origname:
+                origname = spec.name
+
+        actual = dict(vars(spec.loader_state))
+
+        # Check the code object used to import the frozen module.
+        # We can't compare the marshaled data directly because
+        # marshal.dumps() would mark "expected" (below) as a ref,
+        # which slightly changes the output.
+        # (See https://bugs.python.org/issue34093.)
+        data = actual.pop('data')
         with import_helper.frozen_modules():
             expected = _imp.get_frozen_object(spec.name)
-        data = spec.loader_state
-        # We can't compare the marshaled data directly because
-        # marshal.dumps() would mark "expected" as a ref, which slightly
-        # changes the output.  (See https://bugs.python.org/issue34093.)
         code = marshal.loads(data)
         self.assertEqual(code, expected)
 
+        # Check the rest of spec.loader_state.
+        expected = dict(
+            origname=origname,
+        )
+        self.assertDictEqual(actual, expected)
+
     def check_search_locations(self, spec):
         # Frozen packages do not have any path entries.
         # (See https://bugs.python.org/issue21736.)
@@ -58,7 +80,7 @@ def test_module(self):
             with self.subTest(f'{name} -> {name}'):
                 spec = self.find(name)
                 self.check_basic(spec, name)
-                self.check_data(spec)
+                self.check_loader_state(spec)
         modules = {
             '__hello_alias__': '__hello__',
             '_frozen_importlib': 'importlib._bootstrap',
@@ -67,46 +89,50 @@ def test_module(self):
             with self.subTest(f'{name} -> {origname}'):
                 spec = self.find(name)
                 self.check_basic(spec, name)
-                self.check_data(spec)
+                self.check_loader_state(spec, origname)
         modules = [
             '__phello__.__init__',
             '__phello__.ham.__init__',
         ]
         for name in modules:
-            origname = name.rpartition('.')[0]
+            origname = '<' + name.rpartition('.')[0]
+            filename = resolve_stdlib_file(name)
             with self.subTest(f'{name} -> {origname}'):
                    spec = self.find(name)
                 self.check_basic(spec, name)
-                self.check_data(spec)
+                self.check_loader_state(spec, origname, filename)
         modules = {
             '__hello_only__': ('Tools', 'freeze', 'flag.py'),
         }
         for name, path in modules.items():
+            origname = None
             filename = os.path.join(REPO_ROOT, *path)
             with self.subTest(f'{name} -> {filename}'):
                 spec = self.find(name)
                 self.check_basic(spec, name)
-                self.check_data(spec)
+                self.check_loader_state(spec, origname, filename)
 
     def test_package(self):
         packages = [
             '__phello__',
             '__phello__.ham',
         ]
         for name in packages:
+            filename = resolve_stdlib_file(name, ispkg=True)
             with self.subTest(f'{name} -> {name}'):
                 spec = self.find(name)
                 self.check_basic(spec, name, ispkg=True)
-                self.check_data(spec)
+                self.check_loader_state(spec, name, filename)
                 self.check_search_locations(spec)
         packages = {
             '__phello_alias__': '__hello__',
         }
         for name, origname in packages.items():
+            filename = resolve_stdlib_file(origname, ispkg=False)
             with self.subTest(f'{name} -> {origname}'):
                 spec = self.find(name)
                 self.check_basic(spec, name, ispkg=True)
-                self.check_data(spec)
+                self.check_loader_state(spec, origname, filename)
                 self.check_search_locations(spec)
 
     # These are covered by test_module() and test_package().

@@ -32,17 +32,19 @@ def fresh(name, *, oldapi=False):
 
 class ExecModuleTests(abc.LoaderTests):
 
-    def exec_module(self, name):
+    def exec_module(self, name, origname=None):
         with import_helper.frozen_modules():
             is_package = self.machinery.FrozenImporter.is_package(name)
             code = _imp.get_frozen_object(name)
-        data = marshal.dumps(code)
         spec = self.machinery.ModuleSpec(
             name,
             self.machinery.FrozenImporter,
             origin='frozen',
             is_package=is_package,
-            loader_state=data,
+            loader_state=types.SimpleNamespace(
+                data=marshal.dumps(code),
+                origname=origname or name,
+            ),
         )
         module = types.ModuleType(name)
         module.__spec__ = spec
@@ -66,7 +68,8 @@ def test_module(self):
             self.assertEqual(getattr(module, attr), value)
         self.assertEqual(output, 'Hello world!\n')
         self.assertTrue(hasattr(module, '__spec__'))
-        self.assertIsNone(module.__spec__.loader_state)
+        self.assertIsNone(module.__spec__.loader_state.data)
+        self.assertEqual(module.__spec__.loader_state.origname, name)
 
     def test_package(self):
         name = '__phello__'
@@ -79,7 +82,8 @@ def test_package(self):
                                  name=name, attr=attr, given=attr_value,
                                  expected=value))
         self.assertEqual(output, 'Hello world!\n')
-        self.assertIsNone(module.__spec__.loader_state)
+        self.assertIsNone(module.__spec__.loader_state.data)
+        self.assertEqual(module.__spec__.loader_state.origname, name)
 
     def test_lacking_parent(self):
         name = '__phello__.spam'

diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-01-09-06-54.bpo-45020.Cj5VQN.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-01-09-06-54.bpo-45020.Cj5VQN.rst
@@ -0,0 +1,5 @@
+For frozen stdlib modules, record the original module name as
+``module.__spec__.loader_state.origname``.  If the value is different than
+``module.__spec__.name`` then the module was defined as an alias in
+Tools/scripts/freeze_modules.py.  If it is ``None`` then the module comes
+from a source file outside the stdlib.
@@ -9,6 +9,7 @@
 
 #include <Python.h>
 #include <marshal.h>
+#include <pycore_import.h>
 
 #include <stdio.h>
 #include <sys/types.h>
@@ -24,8 +25,12 @@
 static const struct _frozen _PyImport_FrozenModules[] = {
     {0, 0, 0} /* sentinel */
 };
+static const struct _module_alias aliases[] = {
+    {0, 0} /* sentinel */
+};
 
 const struct _frozen *PyImport_FrozenModules;
+const struct _module_alias *_PyImport_FrozenAliases;
 
 static const char header[] =
     "/* Auto-generated by Programs/_freeze_module.c */";
@@ -183,6 +188,7 @@ main(int argc, char *argv[])
     const char *name, *inpath, *outpath;
 
     PyImport_FrozenModules = _PyImport_FrozenModules;
+    _PyImport_FrozenAliases = aliases;
 
     if (argc != 4) {
         fprintf(stderr, "need to specify the name, input and output paths\n");

@@ -36,6 +36,7 @@
    and __phello__.spam.  Loading any will print some famous words... */
 
 #include "Python.h"
+#include "pycore_import.h"
 
 /* Includes for frozen modules: */
 #include "frozen_modules/importlib._bootstrap.h"
@@ -102,9 +103,24 @@ static const struct _frozen _PyImport_FrozenModules[] = {
     {"__phello__.spam", _Py_M____phello___spam,
         (int)sizeof(_Py_M____phello___spam)},
     {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only)},
-    {0, 0, 0} /* sentinel */
+    {0, 0, 0} /* modules sentinel */
 };
 
+static const struct _module_alias aliases[] = {
+    {"_frozen_importlib", "importlib._bootstrap"},
+    {"_frozen_importlib_external", "importlib._bootstrap_external"},
+    {"os.path", "posixpath"},
+    {"__hello_alias__", "__hello__"},
+    {"__phello_alias__", "__hello__"},
+    {"__phello_alias__.spam", "__hello__"},
+    {"__phello__.__init__", "<__phello__"},
+    {"__phello__.ham.__init__", "<__phello__.ham"},
+    {"__hello_only__", NULL},
+    {0, 0} /* aliases sentinel */
+};
+const struct _module_alias *_PyImport_FrozenAliases = aliases;
+
+
 /* Embedding apps may change this pointer to point to their favorite
    collection of frozen modules: */