From 42b6f1abbdb560759e22eb25df3c3e653a8827a7 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sun, 5 May 2019 11:53:11 -0400 Subject: [PATCH 1/5] Add --upgrade-deps to venv module - This allows for pip + setuptools to be automatically upgraded to the latest version on PyPI - Update documentation to represent this change bpo-34556: Add --upgrade to venv module --- Doc/library/venv.rst | 7 ++++++- Doc/using/venv-create.inc | 10 +++++++++- Lib/test/test_venv.py | 41 ++++++++++++++++++++++++++++++++------- Lib/venv/__init__.py | 40 +++++++++++++++++++++++++++++++------- 4 files changed, 82 insertions(+), 16 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 4f083a3181e7a9..240dd1d4f1a9e6 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -97,7 +97,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. class:: EnvBuilder(system_site_packages=False, clear=False, \ symlinks=False, upgrade=False, with_pip=False, \ - prompt=None) + prompt=None, upgrade_deps=False) The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: @@ -123,12 +123,17 @@ creation according to their needs, the :class:`EnvBuilder` class. (defaults to ``None`` which means directory name of the environment would be used). + * ``upgrade_deps`` - Update the base venv modules to the latest on PyPI + .. versionchanged:: 3.4 Added the ``with_pip`` parameter .. versionadded:: 3.6 Added the ``prompt`` parameter + .. versionadded:: 3.8 + Added the ``upgrade_deps`` parameter + Creators of third-party virtual environment tools will be free to use the provided ``EnvBuilder`` class as a base class. diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index 1ada83c07a67f1..ed6dd31e5a6892 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -35,7 +35,7 @@ your :ref:`Python installation `:: The command, if run with ``-h``, will show the available options:: usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] - [--upgrade] [--without-pip] [--prompt PROMPT] + [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] ENV_DIR [ENV_DIR ...] Creates virtual Python environments in one or more target directories. @@ -60,10 +60,18 @@ The command, if run with ``-h``, will show the available options:: environment (pip is bootstrapped by default) --prompt PROMPT Provides an alternative prompt prefix for this environment. + --upgrade-deps Upgrade core dependencies: pip setuptools to the + latest version in PyPI Once an environment has been created, you may wish to activate it, e.g. by sourcing an activate script in its bin directory. +.. versionchanged:: 3.8 + Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI + +.. versionchanged:: 3.6 + Add ability to change the activated venv's prompt via ``--prompt`` + .. versionchanged:: 3.4 Installs pip by default, added the ``--without-pip`` and ``--copies`` options diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 24d3a69b1878b5..9347881d4589dd 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -16,9 +16,9 @@ from test.support import (captured_stdout, captured_stderr, requires_zlib, can_symlink, EnvironmentVarGuard, rmtree, import_module) -import threading import unittest import venv +from unittest.mock import patch try: import ctypes @@ -32,17 +32,21 @@ or sys.prefix == sys.base_prefix, 'cannot run venv.create from within a venv on this platform') + def check_output(cmd, encoding=None): - p = subprocess.Popen(cmd, + p = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding=encoding) + encoding=encoding + ) out, err = p.communicate() if p.returncode: raise subprocess.CalledProcessError( p.returncode, cmd, out, err) return out, err + class BaseTest(unittest.TestCase): """Base class for venv tests.""" maxDiff = 80 * 50 @@ -77,6 +81,7 @@ def get_text_file_contents(self, *args): result = f.read() return result + class BasicTest(BaseTest): """Test venv module functionality.""" @@ -131,12 +136,33 @@ def test_prompt(self): self.assertEqual(context.prompt, '(My prompt) ') self.assertIn("prompt = 'My prompt'\n", data) + def test_upgrade_dependencies(self): + builder = venv.EnvBuilder() + pip_exe = 'pip.exe' if sys.platform == 'win32' else 'pip' + with tempfile.TemporaryDirectory() as fake_env_dir: + + def pip_cmd_checker(cmd): + self.assertEqual( + cmd, + [ + os.path.join(fake_env_dir, pip_exe), + 'install', + '-U', + 'pip', + 'setuptools' + ] + ) + + fake_context = builder.ensure_directories(fake_env_dir) + with patch("venv.subprocess.check_call", pip_cmd_checker): + builder.upgrade_dependencies(fake_context, fake_env_dir) + @requireVenvCreate def test_prefixes(self): """ Test that the prefix values are as expected. """ - #check our prefixes + # check our prefixes self.assertEqual(sys.base_prefix, sys.prefix) self.assertEqual(sys.base_exec_prefix, sys.exec_prefix) @@ -149,7 +175,8 @@ def test_prefixes(self): ('prefix', self.env_dir), ('prefix', self.env_dir), ('base_prefix', sys.prefix), - ('base_exec_prefix', sys.exec_prefix)): + ('base_exec_prefix', sys.exec_prefix) + ): cmd[2] = 'import sys; print(sys.%s)' % prefix out, err = check_output(cmd) self.assertEqual(out.strip(), expected.encode()) @@ -209,7 +236,7 @@ def clear_directory(self, path): rmtree(fn) def test_unoverwritable_fails(self): - #create a file clashing with directories in the env dir + # create a file clashing with directories in the env dir for paths in self.ENV_SUBDIRS[:3]: fn = os.path.join(self.env_dir, *paths) with open(fn, 'wb') as f: @@ -304,7 +331,6 @@ def test_unicode_in_batch_file(self): builder = venv.EnvBuilder(clear=True) builder.create(env_dir) activate = os.path.join(env_dir, self.bindir, 'activate.bat') - envpy = os.path.join(env_dir, self.bindir, self.exe) out, err = check_output( [activate, '&', self.exe, '-c', 'print(0)'], encoding='oem', @@ -466,5 +492,6 @@ def test_with_pip(self): self.do_test_with_pip(False) self.do_test_with_pip(True) + if __name__ == "__main__": unittest.main() diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 4a49b240b8e217..84a56a50625fa7 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -12,6 +12,8 @@ import sysconfig import types + +CORE_VENV_DEPS = ('pip', 'setuptools') logger = logging.getLogger(__name__) @@ -38,16 +40,19 @@ class EnvBuilder: :param with_pip: If True, ensure pip is installed in the virtual environment :param prompt: Alternative terminal prefix for the environment. + :param upgrade_deps: Update the base venv modules to the latest on PyPI """ def __init__(self, system_site_packages=False, clear=False, - symlinks=False, upgrade=False, with_pip=False, prompt=None): + symlinks=False, upgrade=False, with_pip=False, prompt=None, + upgrade_deps=False): self.system_site_packages = system_site_packages self.clear = clear self.symlinks = symlinks self.upgrade = upgrade self.with_pip = with_pip self.prompt = prompt + self.upgrade_deps = upgrade_deps def create(self, env_dir): """ @@ -74,6 +79,8 @@ def create(self, env_dir): # restore it and rewrite the configuration self.system_site_packages = True self.create_configuration(context) + if self.upgrade_deps: + self.upgrade_dependencies(context, env_dir) def clear_directory(self, path): for fn in os.listdir(path): @@ -105,7 +112,6 @@ def create_if_needed(d): prompt = self.prompt if self.prompt is not None else context.env_name context.prompt = '(%s) ' % prompt create_if_needed(env_dir) - env = os.environ executable = getattr(sys, '_base_executable', sys.executable) dirname, exename = os.path.split(os.path.abspath(executable)) context.executable = executable @@ -250,7 +256,7 @@ def setup_python(self, context): if sysconfig.is_python_build(True): # copy init.tcl - for root, dirs, files in os.walk(context.python_dir): + for root, _dirs, files in os.walk(context.python_dir): if 'init.tcl' in files: tcldir = os.path.basename(root) tcldir = os.path.join(context.env_dir, 'Lib', tcldir) @@ -329,11 +335,11 @@ def install_scripts(self, context, path): binpath = context.bin_path plen = len(path) for root, dirs, files in os.walk(path): - if root == path: # at top-level, remove irrelevant dirs + if root == path: # at top-level, remove irrelevant dirs for d in dirs[:]: if d not in ('common', os.name): dirs.remove(d) - continue # ignore files in top level + continue # ignore files in top level for f in files: if (os.name == 'nt' and f.startswith('python') and f.endswith(('.exe', '.pdb'))): @@ -363,15 +369,28 @@ def install_scripts(self, context, path): f.write(data) shutil.copymode(srcfile, dstfile) + def upgrade_dependencies(self, context, env_dir): + logging.debug( + 'Upgrading {} packages in {}'.format(CORE_VENV_DEPS, env_dir) + ) + if sys.platform == 'win32': + pip_exe = os.path.join(context.bin_dir, 'pip.exe') + else: + pip_exe = os.path.join(context.bin_dir, 'pip') + cmd = [pip_exe, 'install', '-U'] + cmd.extend(CORE_VENV_DEPS) + subprocess.check_call(cmd) + def create(env_dir, system_site_packages=False, clear=False, - symlinks=False, with_pip=False, prompt=None): + symlinks=False, with_pip=False, prompt=None): """Create a virtual environment in a directory.""" builder = EnvBuilder(system_site_packages=system_site_packages, clear=clear, symlinks=symlinks, with_pip=with_pip, prompt=prompt) builder.create(env_dir) + def main(args=None): compatible = True if sys.version_info < (3, 3): @@ -432,6 +451,11 @@ def main(args=None): parser.add_argument('--prompt', help='Provides an alternative prompt prefix for ' 'this environment.') + parser.add_argument('--upgrade-deps', default=False, action='store_true', + dest='upgrade_deps', + help='Upgrade core dependencies: {} to the latest ' + 'version in PyPI'.format( + ' '.join(CORE_VENV_DEPS))) options = parser.parse_args(args) if options.upgrade and options.clear: raise ValueError('you cannot supply --upgrade and --clear together.') @@ -440,10 +464,12 @@ def main(args=None): symlinks=options.symlinks, upgrade=options.upgrade, with_pip=options.with_pip, - prompt=options.prompt) + prompt=options.prompt, + upgrade_deps=options.upgrade_deps) for d in options.dirs: builder.create(d) + if __name__ == '__main__': rc = 1 try: From 9bd89f414d5be42e45cb827a92c05f596084b002 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sun, 5 May 2019 14:23:38 -0400 Subject: [PATCH 2/5] Fix bin_dir to bin_path + fix unittest pathing --- Lib/test/test_venv.py | 5 +++-- Lib/venv/__init__.py | 10 +++++----- .../2019-05-05-18-09-40.bpo-34556.o9kfpu.rst | 1 + 3 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-05-05-18-09-40.bpo-34556.o9kfpu.rst diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 9347881d4589dd..4d2a8172af76db 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -138,6 +138,7 @@ def test_prompt(self): def test_upgrade_dependencies(self): builder = venv.EnvBuilder() + bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' pip_exe = 'pip.exe' if sys.platform == 'win32' else 'pip' with tempfile.TemporaryDirectory() as fake_env_dir: @@ -145,7 +146,7 @@ def pip_cmd_checker(cmd): self.assertEqual( cmd, [ - os.path.join(fake_env_dir, pip_exe), + os.path.join(fake_env_dir, bin_path, pip_exe), 'install', '-U', 'pip', @@ -155,7 +156,7 @@ def pip_cmd_checker(cmd): fake_context = builder.ensure_directories(fake_env_dir) with patch("venv.subprocess.check_call", pip_cmd_checker): - builder.upgrade_dependencies(fake_context, fake_env_dir) + builder.upgrade_dependencies(fake_context) @requireVenvCreate def test_prefixes(self): diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 84a56a50625fa7..ba2e887d120ca1 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -80,7 +80,7 @@ def create(self, env_dir): self.system_site_packages = True self.create_configuration(context) if self.upgrade_deps: - self.upgrade_dependencies(context, env_dir) + self.upgrade_dependencies(context) def clear_directory(self, path): for fn in os.listdir(path): @@ -369,14 +369,14 @@ def install_scripts(self, context, path): f.write(data) shutil.copymode(srcfile, dstfile) - def upgrade_dependencies(self, context, env_dir): + def upgrade_dependencies(self, context): logging.debug( - 'Upgrading {} packages in {}'.format(CORE_VENV_DEPS, env_dir) + 'Upgrading {} packages in {}'.format(CORE_VENV_DEPS, context.bin_path) ) if sys.platform == 'win32': - pip_exe = os.path.join(context.bin_dir, 'pip.exe') + pip_exe = os.path.join(context.bin_path, 'pip.exe') else: - pip_exe = os.path.join(context.bin_dir, 'pip') + pip_exe = os.path.join(context.bin_path, 'pip') cmd = [pip_exe, 'install', '-U'] cmd.extend(CORE_VENV_DEPS) subprocess.check_call(cmd) diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-05-18-09-40.bpo-34556.o9kfpu.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-05-18-09-40.bpo-34556.o9kfpu.rst new file mode 100644 index 00000000000000..6eedddcf0e83da --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-05-18-09-40.bpo-34556.o9kfpu.rst @@ -0,0 +1 @@ +Add --upgrade-deps to venv module From 63c4107e846817918b33ec41fa073a2c4ddb84b5 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sun, 5 May 2019 17:40:03 -0400 Subject: [PATCH 3/5] Using logger.debug and not a new raw logging.debug in venv/__init__.py --- Lib/test/test_venv.py | 2 +- Lib/venv/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 4d2a8172af76db..f1debaddaf53aa 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -155,7 +155,7 @@ def pip_cmd_checker(cmd): ) fake_context = builder.ensure_directories(fake_env_dir) - with patch("venv.subprocess.check_call", pip_cmd_checker): + with patch('venv.subprocess.check_call', pip_cmd_checker): builder.upgrade_dependencies(fake_context) @requireVenvCreate diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index ba2e887d120ca1..c3d6d542c942b3 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -370,7 +370,7 @@ def install_scripts(self, context, path): shutil.copymode(srcfile, dstfile) def upgrade_dependencies(self, context): - logging.debug( + logger.debug( 'Upgrading {} packages in {}'.format(CORE_VENV_DEPS, context.bin_path) ) if sys.platform == 'win32': From c35c4696e148c5dede23078c554a85a14fd4ea64 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Tue, 7 May 2019 13:09:11 -0400 Subject: [PATCH 4/5] - Revert all formatting and lint changes - Revert documentation fixes - Add update_deps param to create() function - Update NEWS file correctly --- Doc/library/venv.rst | 2 +- Doc/using/venv-create.inc | 3 --- Lib/test/test_venv.py | 18 ++++++------------ Lib/venv/__init__.py | 12 +++++------- .../2019-05-05-18-09-40.bpo-34556.o9kfpu.rst | 2 +- 5 files changed, 13 insertions(+), 24 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 240dd1d4f1a9e6..d3d5ae2b007d5f 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -123,7 +123,7 @@ creation according to their needs, the :class:`EnvBuilder` class. (defaults to ``None`` which means directory name of the environment would be used). - * ``upgrade_deps`` - Update the base venv modules to the latest on PyPI + * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI .. versionchanged:: 3.4 Added the ``with_pip`` parameter diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index ed6dd31e5a6892..8fd107b332026e 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -69,9 +69,6 @@ The command, if run with ``-h``, will show the available options:: .. versionchanged:: 3.8 Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI -.. versionchanged:: 3.6 - Add ability to change the activated venv's prompt via ``--prompt`` - .. versionchanged:: 3.4 Installs pip by default, added the ``--without-pip`` and ``--copies`` options diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index f1debaddaf53aa..4f6c11b2663efd 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -32,21 +32,17 @@ or sys.prefix == sys.base_prefix, 'cannot run venv.create from within a venv on this platform') - def check_output(cmd, encoding=None): - p = subprocess.Popen( - cmd, + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding=encoding - ) + encoding=encoding) out, err = p.communicate() if p.returncode: raise subprocess.CalledProcessError( p.returncode, cmd, out, err) return out, err - class BaseTest(unittest.TestCase): """Base class for venv tests.""" maxDiff = 80 * 50 @@ -81,7 +77,6 @@ def get_text_file_contents(self, *args): result = f.read() return result - class BasicTest(BaseTest): """Test venv module functionality.""" @@ -163,7 +158,7 @@ def test_prefixes(self): """ Test that the prefix values are as expected. """ - # check our prefixes + #check our prefixes self.assertEqual(sys.base_prefix, sys.prefix) self.assertEqual(sys.base_exec_prefix, sys.exec_prefix) @@ -176,8 +171,7 @@ def test_prefixes(self): ('prefix', self.env_dir), ('prefix', self.env_dir), ('base_prefix', sys.prefix), - ('base_exec_prefix', sys.exec_prefix) - ): + ('base_exec_prefix', sys.exec_prefix)): cmd[2] = 'import sys; print(sys.%s)' % prefix out, err = check_output(cmd) self.assertEqual(out.strip(), expected.encode()) @@ -237,7 +231,7 @@ def clear_directory(self, path): rmtree(fn) def test_unoverwritable_fails(self): - # create a file clashing with directories in the env dir + #create a file clashing with directories in the env dir for paths in self.ENV_SUBDIRS[:3]: fn = os.path.join(self.env_dir, *paths) with open(fn, 'wb') as f: @@ -332,6 +326,7 @@ def test_unicode_in_batch_file(self): builder = venv.EnvBuilder(clear=True) builder.create(env_dir) activate = os.path.join(env_dir, self.bindir, 'activate.bat') + envpy = os.path.join(env_dir, self.bindir, self.exe) out, err = check_output( [activate, '&', self.exe, '-c', 'print(0)'], encoding='oem', @@ -493,6 +488,5 @@ def test_with_pip(self): self.do_test_with_pip(False) self.do_test_with_pip(True) - if __name__ == "__main__": unittest.main() diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index c3d6d542c942b3..c975f731980638 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -256,7 +256,7 @@ def setup_python(self, context): if sysconfig.is_python_build(True): # copy init.tcl - for root, _dirs, files in os.walk(context.python_dir): + for root, dirs, files in os.walk(context.python_dir): if 'init.tcl' in files: tcldir = os.path.basename(root) tcldir = os.path.join(context.env_dir, 'Lib', tcldir) @@ -335,11 +335,11 @@ def install_scripts(self, context, path): binpath = context.bin_path plen = len(path) for root, dirs, files in os.walk(path): - if root == path: # at top-level, remove irrelevant dirs + if root == path: # at top-level, remove irrelevant dirs for d in dirs[:]: if d not in ('common', os.name): dirs.remove(d) - continue # ignore files in top level + continue # ignore files in top level for f in files: if (os.name == 'nt' and f.startswith('python') and f.endswith(('.exe', '.pdb'))): @@ -383,14 +383,13 @@ def upgrade_dependencies(self, context): def create(env_dir, system_site_packages=False, clear=False, - symlinks=False, with_pip=False, prompt=None): + symlinks=False, with_pip=False, prompt=None, upgrade_deps=False): """Create a virtual environment in a directory.""" builder = EnvBuilder(system_site_packages=system_site_packages, clear=clear, symlinks=symlinks, with_pip=with_pip, - prompt=prompt) + prompt=prompt, upgrade_deps=upgrade_deps) builder.create(env_dir) - def main(args=None): compatible = True if sys.version_info < (3, 3): @@ -469,7 +468,6 @@ def main(args=None): for d in options.dirs: builder.create(d) - if __name__ == '__main__': rc = 1 try: diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-05-18-09-40.bpo-34556.o9kfpu.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-05-18-09-40.bpo-34556.o9kfpu.rst index 6eedddcf0e83da..7861eac5cb2560 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2019-05-05-18-09-40.bpo-34556.o9kfpu.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-05-18-09-40.bpo-34556.o9kfpu.rst @@ -1 +1 @@ -Add --upgrade-deps to venv module +Add ``--upgrade-deps`` to venv module. Patch by Cooper Ry Lees From e39f6e5ef69b223683e7a07168594c0cd8aac6c7 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Wed, 8 May 2019 14:02:28 -0400 Subject: [PATCH 5/5] Move logging to f-string --- Lib/venv/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index c975f731980638..b64125fa4fe175 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -371,7 +371,7 @@ def install_scripts(self, context, path): def upgrade_dependencies(self, context): logger.debug( - 'Upgrading {} packages in {}'.format(CORE_VENV_DEPS, context.bin_path) + f'Upgrading {CORE_VENV_DEPS} packages in {context.bin_path}' ) if sys.platform == 'win32': pip_exe = os.path.join(context.bin_path, 'pip.exe')