From f889dc64eb23d15c5160381f4f2b6c2040cd9d5c Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 18 Aug 2023 22:55:20 +0000 Subject: [PATCH 1/8] GH-83417: Allow `venv` add a `.gitignore` file to environments Off by default via code but on by default via the CLI, the `.gitignore` file contains `*` which causes the entire directory to be ignored. --- Doc/library/venv.rst | 11 +++++++++-- Lib/test/test_venv.py | 18 ++++++++++++++++-- Lib/venv/__init__.py | 36 ++++++++++++++++++++++++++++++++---- Lib/venv/__main__.py | 2 +- 4 files changed, 58 insertions(+), 9 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 2482441d649790..fac16a252e78c5 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -143,7 +143,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. class:: EnvBuilder(system_site_packages=False, clear=False, \ symlinks=False, upgrade=False, with_pip=False, \ - prompt=None, upgrade_deps=False) + prompt=None, upgrade_deps=False, \*, gitignore=False) The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: @@ -172,6 +172,10 @@ creation according to their needs, the :class:`EnvBuilder` class. * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI + * ``gitignore`` -- a Boolean value which, if true, will create a + ``.gitignore`` file in the target directory, containing ``*`` to have the + environment ignored by git. + .. versionchanged:: 3.4 Added the ``with_pip`` parameter @@ -181,6 +185,9 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionadded:: 3.9 Added the ``upgrade_deps`` parameter + .. versionadded:: 3.13 + Added the ``gitignore`` parameter + Creators of third-party virtual environment tools will be free to use the provided :class:`EnvBuilder` class as a base class. @@ -343,7 +350,7 @@ There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ symlinks=False, with_pip=False, prompt=None, \ - upgrade_deps=False) + upgrade_deps=False, \*, gitignore=False) Create an :class:`EnvBuilder` with the given keyword arguments, and call its :meth:`~EnvBuilder.create` method with the *env_dir* argument. diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 3d19b2b2e905f3..e0fc04f4b31b47 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -137,7 +137,8 @@ def _check_output_of_default_create(self): self.assertIn('executable = %s' % os.path.realpath(sys.executable), data) copies = '' if os.name=='nt' else ' --copies' - cmd = f'command = {sys.executable} -m venv{copies} --without-pip {self.env_dir}' + cmd = (f'command = {sys.executable} -m venv{copies} --without-pip ' + f'--without-gitignore {self.env_dir}') self.assertIn(cmd, data) fn = self.get_env_file(self.bindir, self.exe) if not os.path.exists(fn): # diagnostics for Windows buildbot failures @@ -156,14 +157,16 @@ def test_config_file_command_key(self): ('upgrade', '--upgrade'), ('upgrade_deps', '--upgrade-deps'), ('prompt', '--prompt'), + ('gitignore', '--without-gitignore'), ] + negated_attrs = {'with_pip', 'symlinks', 'gitignore'} for attr, opt in attrs: rmtree(self.env_dir) if not attr: b = venv.EnvBuilder() else: b = venv.EnvBuilder( - **{attr: False if attr in ('with_pip', 'symlinks') else True}) + **{attr: False if attr in negated_attrs else True}) b.upgrade_dependencies = Mock() # avoid pip command to upgrade deps b._setup_pip = Mock() # avoid pip setup self.run_with_capture(b.create, self.env_dir) @@ -586,6 +589,7 @@ def test_zippath_from_non_installed_posix(self): "-m", "venv", "--without-pip", + "--without-gitignore", self.env_dir] # Our fake non-installed python is not fully functional because # it cannot find the extensions. Set PYTHONPATH so it can run the @@ -633,6 +637,16 @@ def test_activate_shell_script_has_no_dos_newlines(self): error_message = f"CR LF found in line {i}" self.assertFalse(line.endswith(b'\r\n'), error_message) + def test_gitignore(self): + """ + Test that a .gitignore file is created when requested. + The file should contain a `*\n` line. + """ + self.run_with_capture(venv.create, self.env_dir, gitignore=True) + file_lines = self.get_text_file_contents('.gitignore').splitlines() + self.assertIn('*', file_lines) + + @requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 2173c9b13e5cf7..69e771f769d38a 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -41,11 +41,13 @@ class EnvBuilder: environment :param prompt: Alternative terminal prefix for the environment. :param upgrade_deps: Update the base venv modules to the latest on PyPI + :param gitignore: Create a .gitignore file in the environment directory + which causes it to be ignored by git. """ def __init__(self, system_site_packages=False, clear=False, symlinks=False, upgrade=False, with_pip=False, prompt=None, - upgrade_deps=False): + upgrade_deps=False, *, gitignore=False): self.system_site_packages = system_site_packages self.clear = clear self.symlinks = symlinks @@ -56,6 +58,7 @@ def __init__(self, system_site_packages=False, clear=False, prompt = os.path.basename(os.getcwd()) self.prompt = prompt self.upgrade_deps = upgrade_deps + self.gitignore = gitignore def create(self, env_dir): """ @@ -66,6 +69,8 @@ def create(self, env_dir): """ env_dir = os.path.abspath(env_dir) context = self.ensure_directories(env_dir) + if self.gitignore: + self._setup_gitignore(context) # See issue 24875. We need system_site_packages to be False # until after pip is installed. true_system_site_packages = self.system_site_packages @@ -210,6 +215,8 @@ def create_configuration(self, context): args.append('--upgrade-deps') if self.orig_prompt is not None: args.append(f'--prompt="{self.orig_prompt}"') + if not self.gitignore: + args.append('--without-gitignore') args.append(context.env_dir) args = ' '.join(args) @@ -278,6 +285,19 @@ def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): shutil.copyfile(src, dst) + def _setup_gitignore(self, context): + """ + Create a .gitignore file in the environment directory. + + The contents of the file cause the entire environment directory to be + ignored by git. + """ + gitignore_path = os.path.join(context.env_dir, '.gitignore') + with open(gitignore_path, 'w', encoding='utf-8') as file: + file.write('# Created by venv; ' + 'see https://docs.python.org/3/library/venv.html\n') + file.write('*\n') + def setup_python(self, context): """ Set up a Python executable in the environment. @@ -461,11 +481,13 @@ def upgrade_dependencies(self, context): def create(env_dir, system_site_packages=False, clear=False, - symlinks=False, with_pip=False, prompt=None, upgrade_deps=False): + symlinks=False, with_pip=False, prompt=None, upgrade_deps=False, + *, gitignore=False): """Create a virtual environment in a directory.""" builder = EnvBuilder(system_site_packages=system_site_packages, clear=clear, symlinks=symlinks, with_pip=with_pip, - prompt=prompt, upgrade_deps=upgrade_deps) + prompt=prompt, upgrade_deps=upgrade_deps, + gitignore=gitignore) builder.create(env_dir) @@ -525,6 +547,11 @@ def main(args=None): dest='upgrade_deps', help=f'Upgrade core dependencies ({", ".join(CORE_VENV_DEPS)}) ' 'to the latest version in PyPI') + parser.add_argument('--without-gitignore', dest='gitignore', + default=True, action='store_false', + help='Skips adding a .gitignore file to the ' + 'environment directory which causes git to ignore ' + 'the environment directory.') options = parser.parse_args(args) if options.upgrade and options.clear: raise ValueError('you cannot supply --upgrade and --clear together.') @@ -534,7 +561,8 @@ def main(args=None): upgrade=options.upgrade, with_pip=options.with_pip, prompt=options.prompt, - upgrade_deps=options.upgrade_deps) + upgrade_deps=options.upgrade_deps, + gitignore=options.gitignore) for d in options.dirs: builder.create(d) diff --git a/Lib/venv/__main__.py b/Lib/venv/__main__.py index 912423e4a78198..88f55439dc210c 100644 --- a/Lib/venv/__main__.py +++ b/Lib/venv/__main__.py @@ -6,5 +6,5 @@ main() rc = 0 except Exception as e: - print('Error: %s' % e, file=sys.stderr) + print('Error:', e, file=sys.stderr) sys.exit(rc) From 1faf4b0dc3ff4d650111d49870e79150edaeaf50 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 18 Aug 2023 22:58:17 +0000 Subject: [PATCH 2/8] Add a news entry --- .../next/Library/2023-08-18-22-58-07.gh-issue-83417.61J4yM.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-08-18-22-58-07.gh-issue-83417.61J4yM.rst diff --git a/Misc/NEWS.d/next/Library/2023-08-18-22-58-07.gh-issue-83417.61J4yM.rst b/Misc/NEWS.d/next/Library/2023-08-18-22-58-07.gh-issue-83417.61J4yM.rst new file mode 100644 index 00000000000000..3b4f956a27fe4c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-18-22-58-07.gh-issue-83417.61J4yM.rst @@ -0,0 +1,3 @@ +Add the ability for venv to create a ``.gitignore`` file which causes the +created environment to be ignored by git. It is on by default when venv is +called via its CLI. From 31d0558748a1a4f55d6705e07be7ca21007d18d8 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 21 Aug 2023 15:11:14 -0700 Subject: [PATCH 3/8] Apply suggestions from code review Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/venv.rst | 4 ++-- Lib/test/test_venv.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index fac16a252e78c5..5a488831fb4e04 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -143,7 +143,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. class:: EnvBuilder(system_site_packages=False, clear=False, \ symlinks=False, upgrade=False, with_pip=False, \ - prompt=None, upgrade_deps=False, \*, gitignore=False) + prompt=None, upgrade_deps=False, *, gitignore=False) The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: @@ -350,7 +350,7 @@ There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ symlinks=False, with_pip=False, prompt=None, \ - upgrade_deps=False, \*, gitignore=False) + upgrade_deps=False, *, gitignore=False) Create an :class:`EnvBuilder` with the given keyword arguments, and call its :meth:`~EnvBuilder.create` method with the *env_dir* argument. diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index e0fc04f4b31b47..9bd6bea5f7b63d 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -166,7 +166,7 @@ def test_config_file_command_key(self): b = venv.EnvBuilder() else: b = venv.EnvBuilder( - **{attr: False if attr in negated_attrs else True}) + **{attr: attr not in negated_attrs}) b.upgrade_dependencies = Mock() # avoid pip command to upgrade deps b._setup_pip = Mock() # avoid pip setup self.run_with_capture(b.create, self.env_dir) From ce176844ace547f083f565a552fa65b1812ad73b Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 3 Sep 2023 17:24:58 -0700 Subject: [PATCH 4/8] Switch to `scm_ignore_file` --- Doc/library/venv.rst | 24 +++++++++---- Doc/using/venv-create.inc | 73 ++++++++++++++++++++++----------------- Lib/test/test_venv.py | 72 +++++++++++++++++++------------------- Lib/venv/__init__.py | 35 ++++++++++--------- 4 files changed, 115 insertions(+), 89 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 5a488831fb4e04..02d9dc51fb9924 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -143,7 +143,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. class:: EnvBuilder(system_site_packages=False, clear=False, \ symlinks=False, upgrade=False, with_pip=False, \ - prompt=None, upgrade_deps=False, *, gitignore=False) + prompt=None, upgrade_deps=False, *, scm_ignore_file=None) The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: @@ -172,9 +172,11 @@ creation according to their needs, the :class:`EnvBuilder` class. * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI - * ``gitignore`` -- a Boolean value which, if true, will create a - ``.gitignore`` file in the target directory, containing ``*`` to have the - environment ignored by git. + * ``scm_ignore_file`` -- Create an ignore file for the specified source + control manager (SCM). Support is defined by having a method named + ``create_{scm}_ignore_file``. The only value currently supported is + ``"git"`` via :meth:`create_git_ignore_file`. + .. versionchanged:: 3.4 Added the ``with_pip`` parameter @@ -186,7 +188,7 @@ creation according to their needs, the :class:`EnvBuilder` class. Added the ``upgrade_deps`` parameter .. versionadded:: 3.13 - Added the ``gitignore`` parameter + Added the ``scm_ignore_file`` parameter Creators of third-party virtual environment tools will be free to use the provided :class:`EnvBuilder` class as a base class. @@ -346,11 +348,18 @@ creation according to their needs, the :class:`EnvBuilder` class. The directories are allowed to exist (for when an existing environment is being upgraded). + .. method:: create_git_ignore_file(context) + + Creates a ``.gitignore`` file within the virtual environment that causes + the entire directory to be ignored by the ``git`` source control manager. + + .. versionadded:: 3.13 + There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ symlinks=False, with_pip=False, prompt=None, \ - upgrade_deps=False, *, gitignore=False) + upgrade_deps=False, *, scm_ignore_file=None) Create an :class:`EnvBuilder` with the given keyword arguments, and call its :meth:`~EnvBuilder.create` method with the *env_dir* argument. @@ -366,6 +375,9 @@ There is also a module-level convenience function: .. versionchanged:: 3.9 Added the ``upgrade_deps`` parameter + .. versionchanged:: 3.13 + Added the ``scm_ignore_file`` parameter + An example of extending ``EnvBuilder`` -------------------------------------- diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index 2fc90126482268..1cf438b198a9af 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -35,37 +35,48 @@ your :ref:`Python installation `:: The command, if run with ``-h``, will show the available options:: - usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] - [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] - ENV_DIR [ENV_DIR ...] - - Creates virtual Python environments in one or more target directories. - - positional arguments: - ENV_DIR A directory to create the environment in. - - optional arguments: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Try to use symlinks rather than copies, when symlinks - are not the default for the platform. - --copies Try to use copies rather than symlinks, even when - symlinks are the default for the platform. - --clear Delete the contents of the environment directory if it - already exists, before environment creation. - --upgrade Upgrade the environment directory to use this version - of Python, assuming Python has been upgraded in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) - --prompt PROMPT Provides an alternative prompt prefix for this - environment. - --upgrade-deps Upgrade core dependencies (pip) to the - latest version in PyPI - - Once an environment has been created, you may wish to activate it, e.g. by - sourcing an activate script in its bin directory. + usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] + [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] + [--without-scm-ignore-file] + ENV_DIR [ENV_DIR ...] + + Creates virtual Python environments in one or more target directories. + + positional arguments: + ENV_DIR A directory to create the environment in. + + options: + -h, --help show this help message and exit + --system-site-packages + Give the virtual environment access to the system + site-packages dir. + --symlinks Try to use symlinks rather than copies, when + symlinks are not the default for the platform. + --copies Try to use copies rather than symlinks, even when + symlinks are the default for the platform. + --clear Delete the contents of the environment directory if + it already exists, before environment creation. + --upgrade Upgrade the environment directory to use this + version of Python, assuming Python has been upgraded + in-place. + --without-pip Skips installing or upgrading pip in the virtual + environment (pip is bootstrapped by default) + --prompt PROMPT Provides an alternative prompt prefix for this + environment. + --upgrade-deps Upgrade core dependencies (pip) to the latest + version in PyPI + --without-scm-ignore-file + Skips adding the default SCM ignore file to the + environment directory (the default is a .gitignore + file). + + Once an environment has been created, you may wish to activate it, e.g. by + sourcing an activate script in its bin directory. + +.. versionchanged:: 3.13 + + ``--without-scm-ignore-file`` was added along with creating an ignore file + for ``git`` by default. .. versionchanged:: 3.12 diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 9bd6bea5f7b63d..5c8fd7939c6d6c 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -138,7 +138,7 @@ def _check_output_of_default_create(self): os.path.realpath(sys.executable), data) copies = '' if os.name=='nt' else ' --copies' cmd = (f'command = {sys.executable} -m venv{copies} --without-pip ' - f'--without-gitignore {self.env_dir}') + f'--without-scm-ignore-file {self.env_dir}') self.assertIn(cmd, data) fn = self.get_env_file(self.bindir, self.exe) if not os.path.exists(fn): # diagnostics for Windows buildbot failures @@ -148,37 +148,37 @@ def _check_output_of_default_create(self): self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn) def test_config_file_command_key(self): - attrs = [ - (None, None), - ('symlinks', '--copies'), - ('with_pip', '--without-pip'), - ('system_site_packages', '--system-site-packages'), - ('clear', '--clear'), - ('upgrade', '--upgrade'), - ('upgrade_deps', '--upgrade-deps'), - ('prompt', '--prompt'), - ('gitignore', '--without-gitignore'), + options = [ + (None, None, None), # Default case. + ('--copies', 'symlinks', False), + ('--without-pip', 'with_pip', False), + ('--system-site-packages', 'system_site_packages', True), + ('--clear', 'clear', True), + ('--upgrade', 'upgrade', True), + ('--upgrade-deps', 'upgrade_deps', True), + ('--prompt', 'prompt', True), + ('--without-scm-ignore-file', 'scm_ignore_file', None), ] - negated_attrs = {'with_pip', 'symlinks', 'gitignore'} - for attr, opt in attrs: - rmtree(self.env_dir) - if not attr: - b = venv.EnvBuilder() - else: - b = venv.EnvBuilder( - **{attr: attr not in negated_attrs}) - b.upgrade_dependencies = Mock() # avoid pip command to upgrade deps - b._setup_pip = Mock() # avoid pip setup - self.run_with_capture(b.create, self.env_dir) - data = self.get_text_file_contents('pyvenv.cfg') - if not attr: - for opt in ('--system-site-packages', '--clear', '--upgrade', - '--upgrade-deps', '--prompt'): - self.assertNotRegex(data, rf'command = .* {opt}') - elif os.name=='nt' and attr=='symlinks': - pass - else: - self.assertRegex(data, rf'command = .* {opt}') + for opt, attr, value in options: + with self.subTest(opt=opt, attr=attr, value=value): + rmtree(self.env_dir) + if not attr: + kwargs = {} + else: + kwargs = {attr: value} + b = venv.EnvBuilder(**kwargs) + b.upgrade_dependencies = Mock() # avoid pip command to upgrade deps + b._setup_pip = Mock() # avoid pip setup + self.run_with_capture(b.create, self.env_dir) + data = self.get_text_file_contents('pyvenv.cfg') + if not attr or opt.endswith('git'): + for opt in ('--system-site-packages', '--clear', '--upgrade', + '--upgrade-deps', '--prompt'): + self.assertNotRegex(data, rf'command = .* {opt}') + elif os.name=='nt' and attr=='symlinks': + pass + else: + self.assertRegex(data, rf'command = .* {opt}') def test_prompt(self): env_name = os.path.split(self.env_dir)[1] @@ -589,7 +589,7 @@ def test_zippath_from_non_installed_posix(self): "-m", "venv", "--without-pip", - "--without-gitignore", + "--without-scm-ignore-file", self.env_dir] # Our fake non-installed python is not fully functional because # it cannot find the extensions. Set PYTHONPATH so it can run the @@ -621,6 +621,7 @@ def test_zippath_from_non_installed_posix(self): out, err = check_output(cmd) self.assertTrue(zip_landmark.encode() in out) + @requireVenvCreate def test_activate_shell_script_has_no_dos_newlines(self): """ Test that the `activate` shell script contains no CR LF. @@ -637,12 +638,13 @@ def test_activate_shell_script_has_no_dos_newlines(self): error_message = f"CR LF found in line {i}" self.assertFalse(line.endswith(b'\r\n'), error_message) - def test_gitignore(self): + @requireVenvCreate + def test_create_git_ignore_file(self): """ - Test that a .gitignore file is created when requested. + Test that a .gitignore file is created. The file should contain a `*\n` line. """ - self.run_with_capture(venv.create, self.env_dir, gitignore=True) + self.run_with_capture(venv.create, self.env_dir, scm_ignore_file='git') file_lines = self.get_text_file_contents('.gitignore').splitlines() self.assertIn('*', file_lines) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 69e771f769d38a..e4166a6ac79e3d 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -41,13 +41,12 @@ class EnvBuilder: environment :param prompt: Alternative terminal prefix for the environment. :param upgrade_deps: Update the base venv modules to the latest on PyPI - :param gitignore: Create a .gitignore file in the environment directory - which causes it to be ignored by git. + :param scm_ignore_file: Create an ignore file for the specified SCM. """ def __init__(self, system_site_packages=False, clear=False, symlinks=False, upgrade=False, with_pip=False, prompt=None, - upgrade_deps=False, *, gitignore=False): + upgrade_deps=False, *, scm_ignore_file=None): self.system_site_packages = system_site_packages self.clear = clear self.symlinks = symlinks @@ -58,7 +57,9 @@ def __init__(self, system_site_packages=False, clear=False, prompt = os.path.basename(os.getcwd()) self.prompt = prompt self.upgrade_deps = upgrade_deps - self.gitignore = gitignore + if scm_ignore_file: + scm_ignore_file = scm_ignore_file.lower() + self.scm_ignore_file = scm_ignore_file def create(self, env_dir): """ @@ -69,8 +70,8 @@ def create(self, env_dir): """ env_dir = os.path.abspath(env_dir) context = self.ensure_directories(env_dir) - if self.gitignore: - self._setup_gitignore(context) + if self.scm_ignore_file: + getattr(self, f"create_{self.scm_ignore_file}_ignore_file")(context) # See issue 24875. We need system_site_packages to be False # until after pip is installed. true_system_site_packages = self.system_site_packages @@ -215,8 +216,8 @@ def create_configuration(self, context): args.append('--upgrade-deps') if self.orig_prompt is not None: args.append(f'--prompt="{self.orig_prompt}"') - if not self.gitignore: - args.append('--without-gitignore') + if not self.scm_ignore_file: + args.append('--without-scm-ignore-file') args.append(context.env_dir) args = ' '.join(args) @@ -285,7 +286,7 @@ def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): shutil.copyfile(src, dst) - def _setup_gitignore(self, context): + def create_git_ignore_file(self, context): """ Create a .gitignore file in the environment directory. @@ -482,12 +483,12 @@ def upgrade_dependencies(self, context): def create(env_dir, system_site_packages=False, clear=False, symlinks=False, with_pip=False, prompt=None, upgrade_deps=False, - *, gitignore=False): + *, scm_ignore_file=None): """Create a virtual environment in a directory.""" builder = EnvBuilder(system_site_packages=system_site_packages, clear=clear, symlinks=symlinks, with_pip=with_pip, prompt=prompt, upgrade_deps=upgrade_deps, - gitignore=gitignore) + scm_ignore_file=scm_ignore_file) builder.create(env_dir) @@ -547,11 +548,11 @@ def main(args=None): dest='upgrade_deps', help=f'Upgrade core dependencies ({", ".join(CORE_VENV_DEPS)}) ' 'to the latest version in PyPI') - parser.add_argument('--without-gitignore', dest='gitignore', - default=True, action='store_false', - help='Skips adding a .gitignore file to the ' - 'environment directory which causes git to ignore ' - 'the environment directory.') + parser.add_argument('--without-scm-ignore-file', dest='scm_ignore_file', + action='store_const', const=None, default='git', + help='Skips adding the default SCM ignore file to the ' + 'environment directory (the default is a ' + '.gitignore file).') options = parser.parse_args(args) if options.upgrade and options.clear: raise ValueError('you cannot supply --upgrade and --clear together.') @@ -562,7 +563,7 @@ def main(args=None): with_pip=options.with_pip, prompt=options.prompt, upgrade_deps=options.upgrade_deps, - gitignore=options.gitignore) + scm_ignore_file=options.scm_ignore_file) for d in options.dirs: builder.create(d) From c3b96e095d79e2862c2ea83b63fe7e0820d3e6b3 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 9 Sep 2023 17:03:02 -0700 Subject: [PATCH 5/8] Make `scm_ignore_files` accept an iterable --- Doc/library/venv.rst | 17 +++--- Lib/test/test_venv.py | 118 ++++++++++++++++++++++++++++++------------ Lib/venv/__init__.py | 33 ++++++------ 3 files changed, 109 insertions(+), 59 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 02d9dc51fb9924..3321a3c452e7db 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -143,7 +143,8 @@ creation according to their needs, the :class:`EnvBuilder` class. .. class:: EnvBuilder(system_site_packages=False, clear=False, \ symlinks=False, upgrade=False, with_pip=False, \ - prompt=None, upgrade_deps=False, *, scm_ignore_file=None) + prompt=None, upgrade_deps=False, + *, scm_ignore_files=frozenset()) The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: @@ -172,10 +173,10 @@ creation according to their needs, the :class:`EnvBuilder` class. * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI - * ``scm_ignore_file`` -- Create an ignore file for the specified source - control manager (SCM). Support is defined by having a method named - ``create_{scm}_ignore_file``. The only value currently supported is - ``"git"`` via :meth:`create_git_ignore_file`. + * ``scm_ignore_files`` -- Create ignore files based for the specified source + control managers (SCM) in the iterable. Support is defined by having a + method named ``create_{scm}_ignore_file``. The only value supported by + default is ``"git"`` via :meth:`create_git_ignore_file`. .. versionchanged:: 3.4 @@ -188,7 +189,7 @@ creation according to their needs, the :class:`EnvBuilder` class. Added the ``upgrade_deps`` parameter .. versionadded:: 3.13 - Added the ``scm_ignore_file`` parameter + Added the ``scm_ignore_files`` parameter Creators of third-party virtual environment tools will be free to use the provided :class:`EnvBuilder` class as a base class. @@ -359,7 +360,7 @@ There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ symlinks=False, with_pip=False, prompt=None, \ - upgrade_deps=False, *, scm_ignore_file=None) + upgrade_deps=False, *, scm_ignore_files=frozenset()) Create an :class:`EnvBuilder` with the given keyword arguments, and call its :meth:`~EnvBuilder.create` method with the *env_dir* argument. @@ -376,7 +377,7 @@ There is also a module-level convenience function: Added the ``upgrade_deps`` parameter .. versionchanged:: 3.13 - Added the ``scm_ignore_file`` parameter + Added the ``scm_ignore_files`` parameter An example of extending ``EnvBuilder`` -------------------------------------- diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index dfddae53b3006e..a894bb10bd04da 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -82,6 +82,13 @@ def setUp(self): def tearDown(self): rmtree(self.env_dir) + def envpy(self, *, real_env_dir=False): + if real_env_dir: + env_dir = os.path.realpath(self.env_dir) + else: + env_dir = self.env_dir + return os.path.join(env_dir, self.bindir, self.exe) + def run_with_capture(self, func, *args, **kwargs): with captured_stdout() as output: with captured_stderr() as error: @@ -139,7 +146,7 @@ def _check_output_of_default_create(self): os.path.realpath(sys.executable), data) copies = '' if os.name=='nt' else ' --copies' cmd = (f'command = {sys.executable} -m venv{copies} --without-pip ' - f'--without-scm-ignore-file {self.env_dir}') + f'--without-scm-ignore-files {self.env_dir}') self.assertIn(cmd, data) fn = self.get_env_file(self.bindir, self.exe) if not os.path.exists(fn): # diagnostics for Windows buildbot failures @@ -158,7 +165,7 @@ def test_config_file_command_key(self): ('--upgrade', 'upgrade', True), ('--upgrade-deps', 'upgrade_deps', True), ('--prompt', 'prompt', True), - ('--without-scm-ignore-file', 'scm_ignore_file', None), + ('--without-scm-ignore-files', 'scm_ignore_files', frozenset()), ] for opt, attr, value in options: with self.subTest(opt=opt, attr=attr, value=value): @@ -246,8 +253,7 @@ def test_prefixes(self): # check a venv's prefixes rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) - envpy = os.path.join(self.env_dir, self.bindir, self.exe) - cmd = [envpy, '-c', None] + cmd = [self.envpy(), '-c', None] for prefix, expected in ( ('prefix', self.env_dir), ('exec_prefix', self.env_dir), @@ -264,8 +270,7 @@ def test_sysconfig(self): """ rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir, symlinks=False) - envpy = os.path.join(self.env_dir, self.bindir, self.exe) - cmd = [envpy, '-c', None] + cmd = [self.envpy(), '-c', None] for call, expected in ( # installation scheme ('get_preferred_scheme("prefix")', 'venv'), @@ -287,8 +292,7 @@ def test_sysconfig_symlinks(self): """ rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir, symlinks=True) - envpy = os.path.join(self.env_dir, self.bindir, self.exe) - cmd = [envpy, '-c', None] + cmd = [self.envpy(), '-c', None] for call, expected in ( # installation scheme ('get_preferred_scheme("prefix")', 'venv'), @@ -427,8 +431,7 @@ def test_executable(self): """ rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) + envpy = self.envpy(real_env_dir=True) out, err = check_output([envpy, '-c', 'import sys; print(sys.executable)']) self.assertEqual(out.strip(), envpy.encode()) @@ -441,8 +444,7 @@ def test_executable_symlinks(self): rmtree(self.env_dir) builder = venv.EnvBuilder(clear=True, symlinks=True) builder.create(self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) + envpy = self.envpy(real_env_dir=True) out, err = check_output([envpy, '-c', 'import sys; print(sys.executable)']) self.assertEqual(out.strip(), envpy.encode()) @@ -457,7 +459,6 @@ def test_unicode_in_batch_file(self): builder = venv.EnvBuilder(clear=True) builder.create(env_dir) activate = os.path.join(env_dir, self.bindir, 'activate.bat') - envpy = os.path.join(env_dir, self.bindir, self.exe) out, err = check_output( [activate, '&', self.exe, '-c', 'print(0)'], encoding='oem', @@ -476,9 +477,7 @@ def test_multiprocessing(self): rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) - out, err = check_output([envpy, '-c', + out, err = check_output([self.envpy(real_env_dir=True), '-c', 'from multiprocessing import Pool; ' 'pool = Pool(1); ' 'print(pool.apply_async("Python".lower).get(3)); ' @@ -494,10 +493,8 @@ def test_multiprocessing_recursion(self): rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) script = os.path.join(TEST_HOME_DIR, '_test_venv_multiprocessing.py') - subprocess.check_call([envpy, script]) + subprocess.check_call([self.envpy(real_env_dir=True), script]) @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') def test_deactivate_with_strict_bash_opts(self): @@ -524,9 +521,7 @@ def test_macos_env(self): builder = venv.EnvBuilder() builder.create(self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) - out, err = check_output([envpy, '-c', + out, err = check_output([self.envpy(real_env_dir=True), '-c', 'import os; print("__PYVENV_LAUNCHER__" in os.environ)']) self.assertEqual(out.strip(), 'False'.encode()) @@ -588,7 +583,7 @@ def test_zippath_from_non_installed_posix(self): "-m", "venv", "--without-pip", - "--without-scm-ignore-file", + "--without-scm-ignore-files", self.env_dir] # Our fake non-installed python is not fully functional because # it cannot find the extensions. Set PYTHONPATH so it can run the @@ -613,10 +608,9 @@ def test_zippath_from_non_installed_posix(self): # prevent https://github.com/python/cpython/issues/104839 child_env["ASAN_OPTIONS"] = asan_options subprocess.check_call(cmd, env=child_env) - envpy = os.path.join(self.env_dir, self.bindir, self.exe) # Now check the venv created from the non-installed python has # correct zip path in pythonpath. - cmd = [envpy, '-S', '-c', 'import sys; print(sys.path)'] + cmd = [self.envpy(), '-S', '-c', 'import sys; print(sys.path)'] out, err = check_output(cmd) self.assertTrue(zip_landmark.encode() in out) @@ -638,23 +632,79 @@ def test_activate_shell_script_has_no_dos_newlines(self): self.assertFalse(line.endswith(b'\r\n'), error_message) @requireVenvCreate - def test_create_git_ignore_file(self): + def test_scm_ignore_files_git(self): """ - Test that a .gitignore file is created. + Test that a .gitignore file is created when "git" is specified. The file should contain a `*\n` line. """ - self.run_with_capture(venv.create, self.env_dir, scm_ignore_file='git') + self.run_with_capture(venv.create, self.env_dir, + scm_ignore_files={'git'}) file_lines = self.get_text_file_contents('.gitignore').splitlines() self.assertIn('*', file_lines) + @requireVenvCreate + def test_create_scm_ignore_files_multiple(self): + """ + Test that ``scm_ignore_files`` can work with multiple SCMs. + """ + bzrignore_name = ".bzrignore" + contents = "# For Bazaar.\n*\n" + + class BzrEnvBuilder(venv.EnvBuilder): + def create_bzr_ignore_file(self, context): + gitignore_path = os.path.join(context.env_dir, bzrignore_name) + with open(gitignore_path, 'w', encoding='utf-8') as file: + file.write(contents) + + builder = BzrEnvBuilder(scm_ignore_files={'git', 'bzr'}) + self.run_with_capture(builder.create, self.env_dir) + + gitignore_lines = self.get_text_file_contents('.gitignore').splitlines() + self.assertIn('*', gitignore_lines) + + bzrignore = self.get_text_file_contents(bzrignore_name) + self.assertEqual(bzrignore, contents) + + @requireVenvCreate + def test_create_scm_ignore_files_empty(self): + """ + Test that no default ignore files are created when ``scm_ignore_files`` + is empty. + """ + # scm_ignore_files is set to frozenset() by default. + self.run_with_capture(venv.create, self.env_dir) + with self.assertRaises(FileNotFoundError): + self.get_text_file_contents('.gitignore') + + self.assertIn("--without-scm-ignore-files", + self.get_text_file_contents('pyvenv.cfg')) + + @requireVenvCreate + def test_cli_with_scm_ignore_files(self): + """ + Test that default SCM ignore files are created by default via the CLI. + """ + self.run_with_capture(venv.main, ['--without-pip', self.env_dir]) + + gitignore_lines = self.get_text_file_contents('.gitignore').splitlines() + self.assertIn('*', gitignore_lines) + + @requireVenvCreate + def test_cli_without_scm_ignore_files(self): + """ + Test that ``--without-scm-ignore-files`` doesn't create SCM ignore files. + """ + args = ['--without-pip', '--without-scm-ignore-files', self.env_dir] + self.run_with_capture(venv.main, args) + + with self.assertRaises(FileNotFoundError): + self.get_text_file_contents('.gitignore') @requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" def assert_pip_not_installed(self): - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) - out, err = check_output([envpy, '-c', + out, err = check_output([self.envpy(real_env_dir=True), '-c', 'try:\n import pip\nexcept ImportError:\n print("OK")']) # We force everything to text, so unittest gives the detailed diff # if we get unexpected results @@ -721,9 +771,9 @@ def do_test_with_pip(self, system_site_packages): system_site_packages=system_site_packages, with_pip=True) # Ensure pip is available in the virtual environment - envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe) # Ignore DeprecationWarning since pip code is not part of Python - out, err = check_output([envpy, '-W', 'ignore::DeprecationWarning', + out, err = check_output([self.envpy(real_env_dir=True), + '-W', 'ignore::DeprecationWarning', '-W', 'ignore::ImportWarning', '-I', '-m', 'pip', '--version']) # We force everything to text, so unittest gives the detailed diff @@ -744,7 +794,7 @@ def do_test_with_pip(self, system_site_packages): # It seems ensurepip._uninstall calls subprocesses which do not # inherit the interpreter settings. envvars["PYTHONWARNINGS"] = "ignore" - out, err = check_output([envpy, + out, err = check_output([self.envpy(real_env_dir=True), '-W', 'ignore::DeprecationWarning', '-W', 'ignore::ImportWarning', '-I', '-m', 'ensurepip._uninstall']) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index e4166a6ac79e3d..34d8c2cd7ff8e1 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -41,12 +41,13 @@ class EnvBuilder: environment :param prompt: Alternative terminal prefix for the environment. :param upgrade_deps: Update the base venv modules to the latest on PyPI - :param scm_ignore_file: Create an ignore file for the specified SCM. + :param scm_ignore_files: Create ignore files for the SCMs specified by the + iterable. """ def __init__(self, system_site_packages=False, clear=False, symlinks=False, upgrade=False, with_pip=False, prompt=None, - upgrade_deps=False, *, scm_ignore_file=None): + upgrade_deps=False, *, scm_ignore_files=frozenset()): self.system_site_packages = system_site_packages self.clear = clear self.symlinks = symlinks @@ -57,9 +58,7 @@ def __init__(self, system_site_packages=False, clear=False, prompt = os.path.basename(os.getcwd()) self.prompt = prompt self.upgrade_deps = upgrade_deps - if scm_ignore_file: - scm_ignore_file = scm_ignore_file.lower() - self.scm_ignore_file = scm_ignore_file + self.scm_ignore_files = frozenset(map(str.lower, scm_ignore_files)) def create(self, env_dir): """ @@ -70,8 +69,8 @@ def create(self, env_dir): """ env_dir = os.path.abspath(env_dir) context = self.ensure_directories(env_dir) - if self.scm_ignore_file: - getattr(self, f"create_{self.scm_ignore_file}_ignore_file")(context) + for scm in self.scm_ignore_files: + getattr(self, f"create_{scm}_ignore_file")(context) # See issue 24875. We need system_site_packages to be False # until after pip is installed. true_system_site_packages = self.system_site_packages @@ -216,8 +215,8 @@ def create_configuration(self, context): args.append('--upgrade-deps') if self.orig_prompt is not None: args.append(f'--prompt="{self.orig_prompt}"') - if not self.scm_ignore_file: - args.append('--without-scm-ignore-file') + if not self.scm_ignore_files: + args.append('--without-scm-ignore-files') args.append(context.env_dir) args = ' '.join(args) @@ -483,12 +482,12 @@ def upgrade_dependencies(self, context): def create(env_dir, system_site_packages=False, clear=False, symlinks=False, with_pip=False, prompt=None, upgrade_deps=False, - *, scm_ignore_file=None): + *, scm_ignore_files=frozenset()): """Create a virtual environment in a directory.""" builder = EnvBuilder(system_site_packages=system_site_packages, clear=clear, symlinks=symlinks, with_pip=with_pip, prompt=prompt, upgrade_deps=upgrade_deps, - scm_ignore_file=scm_ignore_file) + scm_ignore_files=scm_ignore_files) builder.create(env_dir) @@ -548,11 +547,11 @@ def main(args=None): dest='upgrade_deps', help=f'Upgrade core dependencies ({", ".join(CORE_VENV_DEPS)}) ' 'to the latest version in PyPI') - parser.add_argument('--without-scm-ignore-file', dest='scm_ignore_file', - action='store_const', const=None, default='git', - help='Skips adding the default SCM ignore file to the ' - 'environment directory (the default is a ' - '.gitignore file).') + parser.add_argument('--without-scm-ignore-files', dest='scm_ignore_files', + action='store_const', const=frozenset(), + default=frozenset(['git']), + help='Skips adding SCM ignore files to the environment ' + 'directory (git is supported by default).') options = parser.parse_args(args) if options.upgrade and options.clear: raise ValueError('you cannot supply --upgrade and --clear together.') @@ -563,7 +562,7 @@ def main(args=None): with_pip=options.with_pip, prompt=options.prompt, upgrade_deps=options.upgrade_deps, - scm_ignore_file=options.scm_ignore_file) + scm_ignore_files=options.scm_ignore_files) for d in options.dirs: builder.create(d) From ddc2225c98a8d0240d47f488c51b669600ffb953 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 9 Sep 2023 17:43:17 -0700 Subject: [PATCH 6/8] Update Doc/library/venv.rst Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/venv.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 3321a3c452e7db..b72f3041f1da11 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -143,7 +143,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. class:: EnvBuilder(system_site_packages=False, clear=False, \ symlinks=False, upgrade=False, with_pip=False, \ - prompt=None, upgrade_deps=False, + prompt=None, upgrade_deps=False, \ *, scm_ignore_files=frozenset()) The :class:`EnvBuilder` class accepts the following keyword arguments on From 547879d48bed4c4f8b024e78cceffbdfa5e0f973 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 11 Sep 2023 15:43:59 -0700 Subject: [PATCH 7/8] Fix capitalization of "Git" Co-authored-by: Hugo van Kemenade --- Lib/venv/__init__.py | 2 +- .../next/Library/2023-08-18-22-58-07.gh-issue-83417.61J4yM.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 34d8c2cd7ff8e1..d960bf3bd82ac5 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -551,7 +551,7 @@ def main(args=None): action='store_const', const=frozenset(), default=frozenset(['git']), help='Skips adding SCM ignore files to the environment ' - 'directory (git is supported by default).') + 'directory (Git is supported by default).') options = parser.parse_args(args) if options.upgrade and options.clear: raise ValueError('you cannot supply --upgrade and --clear together.') diff --git a/Misc/NEWS.d/next/Library/2023-08-18-22-58-07.gh-issue-83417.61J4yM.rst b/Misc/NEWS.d/next/Library/2023-08-18-22-58-07.gh-issue-83417.61J4yM.rst index 3b4f956a27fe4c..fbb8bdb2073efa 100644 --- a/Misc/NEWS.d/next/Library/2023-08-18-22-58-07.gh-issue-83417.61J4yM.rst +++ b/Misc/NEWS.d/next/Library/2023-08-18-22-58-07.gh-issue-83417.61J4yM.rst @@ -1,3 +1,3 @@ Add the ability for venv to create a ``.gitignore`` file which causes the -created environment to be ignored by git. It is on by default when venv is +created environment to be ignored by Git. It is on by default when venv is called via its CLI. From f75b26cf3717b1265feb7da6c5c54bf1f411d041 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 15 Sep 2023 15:09:16 -0700 Subject: [PATCH 8/8] Add a What's new entry --- Doc/whatsnew/3.13.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 8c6467562aeb62..a6f50775d64729 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -220,6 +220,16 @@ typing check whether a class is a :class:`typing.Protocol`. (Contributed by Jelle Zijlstra in :gh:`104873`.) +venv +---- + +* Add support for adding source control management (SCM) ignore files to a + virtual environment's directory. By default, Git is supported. This is + implemented as opt-in via the API which can be extended to support other SCMs + (:class:`venv.EnvBuilder` and :func:`venv.create`), and opt-out via the CLI + (using ``--without-scm-ignore-files``). (Contributed by Brett Cannon in + :gh:`108125`.) + Optimizations =============