From 1e52c234091b3dd4649a0e0b8d13724e44d6ecd3 Mon Sep 17 00:00:00 2001 From: Andreas Stocker Date: Sat, 13 Jul 2024 15:47:29 +0200 Subject: [PATCH 1/3] gh-121342: Fixed `pkgutil.iter_zipimport_modules` on an invalidated cache (#121342) It is no longer safe to directly access `zipimport._zip_directory_cache` since #103208. It is not guaranteed that the cache is acutally filled. This changed fixes this by using the internal method `_get_files` instead, which may not be the best solution, but fixes the issue. --- Lib/pkgutil.py | 10 ++++- Lib/test/test_pkgutil.py | 45 +++++++++++++++++++ ...-07-13-15-58-52.gh-issue-121342.NyuGnr.rst | 2 + 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index dccbec52aa731e..2e499f5705a960 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -176,7 +176,15 @@ def _iter_file_finder_modules(importer, prefix=''): from zipimport import zipimporter def iter_zipimport_modules(importer, prefix=''): - dirlist = sorted(zipimport._zip_directory_cache[importer.archive]) + """Iterate through the modules available within a zipfile + + For each module found in the zipfile, a tuple containing the module's + name and boolean indicating whether the module is a package + gets yielded. + + It is assumed that importer is an instance of zipimport.zipimporter. + """ + dirlist = sorted(zipimporter(importer.archive)._get_files()) _prefix = importer.prefix plen = len(_prefix) yielded = {} diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index d095f440a99f63..2566ccca62cb10 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -522,6 +522,51 @@ def test_mixed_namespace(self): del sys.modules['foo.bar'] del sys.modules['foo.baz'] + def test_iter_zipimport_modules(self): + with zipfile.ZipFile( + tempfile.NamedTemporaryFile(suffix='.zip', delete=False), + mode='w' + ) as tmp_file_zip: + tmp_file_zip.writestr( + 'foo.py', + 'print("foot")' + ) + tmp_file_zip.writestr( + 'bar/__init__.py', + 'print("bar")' + ) + + module_zip = pkgutil.zipimporter(tmp_file_zip.filename) + + self.assertIn(('foo', False), pkgutil.iter_zipimport_modules(module_zip, prefix='')) + self.assertIn(('bar', True), pkgutil.iter_zipimport_modules(module_zip, prefix='')) + + # Cleanup + os.remove(tmp_file_zip.filename) + + def test_iter_zipimport_modules_invalidate_caches(self): + with zipfile.ZipFile( + tempfile.NamedTemporaryFile(suffix='.zip', delete=False), + mode='w' + ) as tmp_file_zip: + tmp_file_zip.writestr( + 'foo.py', + 'print("foo")' + ) + tmp_file_zip.writestr( + 'bar/__init__.py', + 'print("bar")' + ) + + module_zip = pkgutil.zipimporter(tmp_file_zip.filename) + module_zip.invalidate_caches() + + self.assertIn(('foo', False), pkgutil.iter_zipimport_modules(module_zip, prefix='')) + self.assertIn(('bar', True), pkgutil.iter_zipimport_modules(module_zip, prefix='')) + + # Cleanup + os.remove(tmp_file_zip.filename) + # XXX: test .pkg files diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst new file mode 100644 index 00000000000000..17209c6d06a9e1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst @@ -0,0 +1,2 @@ +Fix :func:`pkgutil.iter_zipimport_modules` when called on an invalidated +cache. Patch by Andreas Stocker. From bc9c996d5dcb3db5a61aa072fde644d059ca117b Mon Sep 17 00:00:00 2001 From: Andreas Stocker Date: Sun, 14 Jul 2024 12:58:35 +0200 Subject: [PATCH 2/3] gh-121342: Unreferenced `pkgutil.iter_zipimport_modules` in NEWS.d As `pkgutil.iter_zipimport_modules` isn't supposed to be consumed directly by anything other than `pkgutils` itself, and is registered via as an importer module for `zipimporter`, I've decided to not include it in the public documentation. Thus, the `:func:` reference of `pkgutil.iter_zipimport_modules` in the news file is removed now. --- .../2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst index 17209c6d06a9e1..e486ba44d89826 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst @@ -1,2 +1,2 @@ -Fix :func:`pkgutil.iter_zipimport_modules` when called on an invalidated -cache. Patch by Andreas Stocker. +Fix `pkgutil.iter_zipimport_modules` when called on an invalidated cache. +Patch by Andreas Stocker. From 7d2a3fb3ff31b1f7fcc79d5b893ef5a07eadf52d Mon Sep 17 00:00:00 2001 From: Andreas Stocker Date: Sun, 14 Jul 2024 13:09:23 +0200 Subject: [PATCH 3/3] gh-121342: Make the Sphinx linter happy again --- .../2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst index e486ba44d89826..05c1e09e562e3b 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-13-15-58-52.gh-issue-121342.NyuGnr.rst @@ -1,2 +1,2 @@ -Fix `pkgutil.iter_zipimport_modules` when called on an invalidated cache. +Fix ``pkgutil.iter_zipimport_modules`` when called on an invalidated cache. Patch by Andreas Stocker.