8000 bpo-45020: Identify which frozen modules are actually aliases. by ericsnowcurrently · Pull Request #28655 · python/cpython · GitHub
[go: up one dir, main page]

Skip to content
8000

bpo-45020: Identify which frozen modules are actually aliases. #28655

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
7 changes: 7 additions & 0 deletions Include/internal/pycore_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ extern PyStatus _PyImport_ReInitLock(void);
#endif
extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate);

struct _module_alias {
const char *name; /* ASCII encoded string */
const char *orig; /* ASCII encoded string */
};

extern const struct _module_alias * _PyImport_FrozenAliases;

#ifdef __cplusplus
}
#endif
Expand Down
33 changes: 29 additions & 4 deletions Lib/importlib/_bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -824,16 +824,39 @@ def module_repr(m):
"slated for removal in Python 3.12", DeprecationWarning)
return '<module {!r} ({})>'.format(m.__name__, FrozenImporter._ORIGIN)

@classmethod
def _setup_module(cls, module):
assert not hasattr(module, '__file__'), module.__file__
ispkg = hasattr(module, '__path__')
assert not ispkg or not module.__path__, module.__path__
spec = module.__spec__
assert not ispkg or not spec.submodule_search_locations

if spec.loader_state is None:
spec.loader_state = type(sys.implementation)(
data=None,
origname=None,
)
elif not hasattr(spec.loader_state, 'data'):
spec.loader_state.data = None
if not getattr(spec.loader_state, 'origname', None):
origname = vars(module).pop('__origname__', None)
assert origname, 'see PyImport_ImportFrozenModuleObject()'
spec.loader_state.origname = origname

@classmethod
def find_spec(cls, fullname, path=None, target=None):
info = _call_with_frames_removed(_imp.find_frozen, fullname)
if info is None:
return None
data, ispkg = info
data, ispkg, origname = info
spec = spec_from_loader(fullname, cls,
origin=cls._ORIGIN,
is_package=ispkg)
spec.loader_state = data
spec.loader_state = type(sys.implementation)(
data=data,
origname=origname,
)
return spec

@classmethod
Expand All @@ -857,7 +880,7 @@ def exec_module(module):
spec = module.__spec__
name = spec.name
try:
data = spec.loader_state
data = spec.loader_state.data
except AttributeError:
if not _imp.is_frozen(name):
raise ImportError('{!r} is not a frozen module'.format(name),
Expand All @@ -868,7 +891,7 @@ def exec_module(module):
# Note that if this method is called again (e.g. by
# importlib.reload()) then _imp.get_frozen_object() will notice
# no data was provided and will look it up.
spec.loader_state = None
spec.loader_state.data = None
code = _call_with_frames_removed(_imp.get_frozen_object, name, data)
exec(code, module.__dict__)

Expand Down Expand Up @@ -1220,6 +1243,8 @@ def _setup(sys_module, _imp_module):
continue
spec = _spec_from_module(module, loader)
_init_module_attrs(spec, module)
if loader is FrozenImporter:
loader._setup_module(module)

# Directly load built-in modules needed during bootstrap.
self_module = sys.modules[__name__]
Expand Down
52 changes: 39 additions & 13 deletions Lib/test/test_importlib/frozen/test_finder.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,15 @@
import unittest
import warnings

from test.support import import_helper, REPO_ROOT
from test.support import import_helper, REPO_ROOT, STDLIB_DIR


def resolve_stdlib_file(name, ispkg=False):
assert name
if ispkg:
return os.path.join(STDLIB_DIR, *name.split('.'), '__init__.py')
else:
return os.path.join(STDLIB_DIR, *name.split('.')) + '.py'


class FindSpecTests(abc.FinderTests):
Expand All @@ -32,16 +40,30 @@ def check_basic(self, spec, name, ispkg=False):
self.assertIsNone(spec.submodule_search_locations)
self.assertIsNotNone(spec.loader_state)

def check_data(self, spec):
def check_loader_state(self, spec, origname=None, filename=None):
if not filename:
if not origname:
origname = spec.name

actual = dict(vars(spec.loader_state))

# Check the code object used to import the frozen module.
# We can't compare the marshaled data directly because
# marshal.dumps() would mark "expected" (below) as a ref,
# which slightly changes the output.
# (See https://bugs.python.org/issue34093.)
data = actual.pop('data')
with import_helper.frozen_modules():
expected = _imp.get_frozen_object(spec.name)
data = spec.loader_state
# We can't compare the marshaled data directly because
# marshal.dumps() would mark "expected" as a ref, which slightly
# changes the output. (See https://bugs.python.org/issue34093.)
code = marshal.loads(data)
self.assertEqual(code, expected)

# Check the rest of spec.loader_state.
expected = dict(
origname=origname,
)
self.assertDictEqual(actual, expected)

def check_search_locations(self, spec):
# Frozen packages do not have any path entries.
# (See https://bugs.python.org/issue21736.)
Expand All @@ -58,7 +80,7 @@ def test_module(self):
with self.subTest(f'{name} -> {name}'):
spec = self.find(name)
self.check_basic(spec, name)
self.check_data(spec)
self.check_loader_state(spec)
modules = {
'__hello_alias__': '__hello__',
'_frozen_importlib': 'importlib._bootstrap',
Expand All @@ -67,46 +89,50 @@ def test_module(self):
with self.subTest(f'{name} -> {origname}'):
spec = self.find(name)
self.check_basic(spec, name)
self.check_data(spec)
self.check_loader_state(spec, origname)
modules = [
'__phello__.__init__',
'__phello__.ham.__init__',
]
for name in modules:
origname = name.rpartition('.')[0]
origname = '<' + name.rpartition('.')[0]
filename = resolve_stdlib_file(name)
with self.subTest(f'{name} -> {origname}'):
spec = self.find(name)
self.check_basic(spec, name)
self.check_data(spec)
self.check_loader_state(spec, origname, filename)
modules = {
'__hello_only__': ('Tools', 'freeze', 'flag.py'),
}
for name, path in modules.items():
origname = None
filename = os.path.join(REPO_ROOT, *path)
with self.subTest(f'{name} -> {filename}'):
spec = self.find(name)
self.check_basic(spec, name)
self.check_data(spec)
self.check_loader_state(spec, origname, filename)

def test_package(self):
packages = [
'__phello__',
'__phello__.ham',
]
for name in packages:
filename = resolve_stdlib_file(name, ispkg=True)
with self.subTest(f'{name} -> {name}'):
spec = self.find(name)
self.check_basic(spec, name, ispkg=True)
self.check_data(spec)
self.check_loader_state(spec, name, filename)
self.check_search_locations(spec)
packages = {
'__phello_alias__': '__hello__',
}
for name, origname in packages.items():
filename = resolve_stdlib_file(origname, ispkg=False)
with self.subTest(f'{name} -> {origname}'):
spec = self.find(name)
self.check_basic(spec, name, ispkg=True)
self.check_data(spec)
self.check_loader_state(spec, origname, filename)
self.check_search_locations(spec)

# These are covered by test_module() and test_package().
Expand Down
14 changes: 9 additions & 5 deletions Lib/test/test_importlib/frozen/test_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,19 @@ def fresh(name, *, oldapi=False):

class ExecModuleTests(abc.LoaderTests):

def exec_module(self, name):
def exec_module(self, name, origname=None):
with import_helper.frozen_modules():
is_package = self.machinery.FrozenImporter.is_package(name)
code = _imp.get_frozen_object(name)
data = marshal.dumps(code)
spec = self.machinery.ModuleSpec(
name,
self.machinery.FrozenImporter,
origin='frozen',
is_package=is_package,
loader_state=data,
loader_state=types.SimpleNamespace(
data=marshal.dumps(code),
origname=origname or name,
),
)
module = types.ModuleType(name)
module.__spec__ = spec
Expand All @@ -66,7 +68,8 @@ def test_module(self):
self.assertEqual(getattr(module, attr), value)
self.assertEqual(output, 'Hello world!\n')
self.assertTrue(hasattr(module, '__spec__'))
self.assertIsNone(module.__spec__.loader_state)
self.assertIsNone(module.__spec__.loader_state.data)
self.assertEqual(module.__spec__.loader_state.origname, name)

def test_package(self):
name = '__phello__'
Expand All @@ -79,7 +82,8 @@ def test_package(self):
name=name, attr=attr, given=attr_value,
expected=value))
self.assertEqual(output, 'Hello world!\n')
self.assertIsNone(module.__spec__.loader_state)
self.assertIsNone(module.__spec__.loader_state.data)
self.assertEqual(module.__spec__.loader_state.origname, name)

def test_lacking_parent(self):
name = '__phello__.spam'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
For frozen stdlib modules, record the original module name as
``module.__spec__.loader_state.origname``. If the value is different than
``module.__spec__.name`` then the module was defined as an alias in
Tools/scripts/freeze_modules.py. If it is ``None`` then the module comes
from a source file outside the stdlib.
6 changes: 6 additions & 0 deletions Programs/_freeze_module.c
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

#include <Python.h>
#include <marshal.h>
#include <pycore_import.h>

#include <stdio.h>
#include <sys/types.h>
Expand All @@ -24,8 +25,12 @@
static const struct _frozen _PyImport_FrozenModules[] = {
{0, 0, 0} /* sentinel */
};
static const struct _module_alias aliases[] = {
{0, 0} /* sentinel */
};

const struct _frozen *PyImport_FrozenModules;
const struct _module_alias *_PyImport_FrozenAliases;

static const char header[] =
"/* Auto-generated by Programs/_freeze_module.c */";
Expand Down Expand Up @@ -183,6 +188,7 @@ main(int argc, char *argv[])
const char *name, *inpath, *outpath;

PyImport_FrozenModules = _PyImport_FrozenModules;
_PyImport_FrozenAliases = aliases;

if (argc != 4) {
fprintf(stderr, "need to specify the name, input and output paths\n");
Expand Down
7 changes: 5 additions & 2 deletions Python/clinic/import.c.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

18 changes: 17 additions & 1 deletion Python/frozen.c
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
and __phello__.spam. Loading any will print some famous words... */

#include "Python.h"
#include "pycore_import.h"

/* Includes for frozen modules: */
#include "frozen_modules/importlib._bootstrap.h"
Expand Down Expand Up @@ -102,9 +103,24 @@ static const struct _frozen _PyImport_FrozenModules[] = {
{"__phello__.spam", _Py_M____phello___spam,
(int)sizeof(_Py_M____phello___spam)},
{"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only)},
{0, 0, 0} /* sentinel */
{0, 0, 0} /* modules sentinel */
};

static const struct _module_alias aliases[] = {
{"_frozen_importlib", "importlib._bootstrap"},
{"_frozen_importlib_external", "importlib._bootstrap_external"},
{"os.path", "posixpath"},
{"__hello_alias__", "__hello__"},
{"__phello_alias__", "__hello__"},
{"__phello_alias__.spam", "__hello__"},
{"__phello__.__init__", "<__phello__"},
{"__phello__.ham.__init__", "<__phello__.ham"},
{"__hello_only__", NULL},
{0, 0} /* aliases sentinel */
};
const struct _module_alias *_PyImport_FrozenAliases = aliases;


/* Embedding apps may change this pointer to point to their favorite
collection of frozen modules: */

Expand Down
Loading
0