From c08e8c6a75e2aa46f4befb62cdbf8b58a91ea37e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Hendrik=20M=C3=BCller?= <44469195+kolibril13@users.noreply.github.com> Date: Thu, 6 Jan 2022 21:01:46 +0100 Subject: [PATCH 01/85] added second example --- IPython/core/magic_arguments.py | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/IPython/core/magic_arguments.py b/IPython/core/magic_arguments.py index 9231609572e..d147229512b 100644 --- a/IPython/core/magic_arguments.py +++ b/IPython/core/magic_arguments.py @@ -37,6 +37,40 @@ def magic_cool(self, arg): -o OPTION, --option OPTION An optional argument. + +Here is an elaborated example that uses default parameters in `argument` and calls the `args` in the cell magic:: + + from IPython.core.magic import register_cell_magic + from IPython.core.magic_arguments import (argument, magic_arguments, + parse_argstring) + + + @magic_arguments() + @argument( + "--option", + "-o", + help=("Add an option here"), + ) + @argument( + "--style", + "-s", + default="foo", + help=("Add some style arguments"), + ) + @register_cell_magic + def my_cell_magic(line, cell): + args = parse_argstring(my_cell_magic, line) + print(f"{args.option=}") + print(f"{args.style=}") + print(f"{cell=}") + +In a jupyter notebook, this cell magic can be executed like this:: + + %%my_cell_magic -o Hello + print("bar") + i = 42 + + Inheritance diagram: .. inheritance-diagram:: IPython.core.magic_arguments From 6fe9697c3581795145acb0a7a628fdd8e89ec379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Hendrik=20M=C3=BCller?= <44469195+kolibril13@users.noreply.github.com> Date: Thu, 6 Jan 2022 21:46:20 +0100 Subject: [PATCH 02/85] remove blank lines --- IPython/core/magic_arguments.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/IPython/core/magic_arguments.py b/IPython/core/magic_arguments.py index d147229512b..568abd82ae0 100644 --- a/IPython/core/magic_arguments.py +++ b/IPython/core/magic_arguments.py @@ -37,7 +37,6 @@ def magic_cool(self, arg): -o OPTION, --option OPTION An optional argument. - Here is an elaborated example that uses default parameters in `argument` and calls the `args` in the cell magic:: from IPython.core.magic import register_cell_magic @@ -70,7 +69,6 @@ def my_cell_magic(line, cell): print("bar") i = 42 - Inheritance diagram: .. inheritance-diagram:: IPython.core.magic_arguments From cb6563dcd85783ea0a687fbe227c4782a8a7cadf Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 10 Jan 2022 09:28:10 +0100 Subject: [PATCH 03/85] Fix and test for "async with does not allow new lines". Use the opportunity to add a test, and parametrise a few other, plus set the correct stacklevel. Closes #12975 --- IPython/core/inputtransformer2.py | 12 ++--- IPython/core/tests/test_inputtransformer2.py | 46 +++++++++++++------- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py index 85d448a727f..3a560073b22 100644 --- a/IPython/core/inputtransformer2.py +++ b/IPython/core/inputtransformer2.py @@ -507,9 +507,12 @@ def make_tokens_by_line(lines:List[str]): # reexported from token on 3.7+ NEWLINE, NL = tokenize.NEWLINE, tokenize.NL # type: ignore - tokens_by_line:List[List[Any]] = [[]] - if len(lines) > 1 and not lines[0].endswith(('\n', '\r', '\r\n', '\x0b', '\x0c')): - warnings.warn("`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified") + tokens_by_line: List[List[Any]] = [[]] + if len(lines) > 1 and not lines[0].endswith(("\n", "\r", "\r\n", "\x0b", "\x0c")): + warnings.warn( + "`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified", + stacklevel=2, + ) parenlev = 0 try: for token in tokenize.generate_tokens(iter(lines).__next__): @@ -782,9 +785,6 @@ def __init__(self, extra_flags=0): super().__init__() self.flags |= extra_flags - def __call__(self, *args, **kwds): - return compile(*args, **kwds) - class MaybeAsyncCommandCompiler(CommandCompiler): def __init__(self, extra_flags=0): diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py index 2207cdfb48c..bc40721c9ff 100644 --- a/IPython/core/tests/test_inputtransformer2.py +++ b/IPython/core/tests/test_inputtransformer2.py @@ -4,6 +4,7 @@ more complex. See test_inputtransformer2_line for tests for line-based transformations. """ +import platform import string import sys from textwrap import dedent @@ -319,7 +320,16 @@ def test_check_complete(): assert cc("def f():\n x=0\n \\\n ") == ("incomplete", 2) -def test_check_complete_II(): +@pytest.mark.skipif(platform.python_implementation() == "PyPy", reason="fail on pypy") +@pytest.mark.parametrize( + "value, expected", + [ + ('''def foo():\n """''', ("incomplete", 4)), + ("""async with example:\n pass""", ("incomplete", 4)), + ("""async with example:\n pass\n """, ("complete", None)), + ], +) +def test_check_complete_II(value, expected): """ Test that multiple line strings are properly handled. @@ -327,25 +337,31 @@ def test_check_complete_II(): """ cc = ipt2.TransformerManager().check_complete - assert cc('''def foo():\n """''') == ("incomplete", 4) - - -def test_check_complete_invalidates_sunken_brackets(): + assert cc(value) == expected + + +@pytest.mark.parametrize( + "value, expected", + [ + (")", ("invalid", None)), + ("]", ("invalid", None)), + ("}", ("invalid", None)), + (")(", ("invalid", None)), + ("][", ("invalid", None)), + ("}{", ("invalid", None)), + ("]()(", ("invalid", None)), + ("())(", ("invalid", None)), + (")[](", ("invalid", None)), + ("()](", ("invalid", None)), + ], +) +def test_check_complete_invalidates_sunken_brackets(value, expected): """ Test that a single line with more closing brackets than the opening ones is interpreted as invalid """ cc = ipt2.TransformerManager().check_complete - assert cc(")") == ("invalid", None) - assert cc("]") == ("invalid", None) - assert cc("}") == ("invalid", None) - assert cc(")(") == ("invalid", None) - assert cc("][") == ("invalid", None) - assert cc("}{") == ("invalid", None) - assert cc("]()(") == ("invalid", None) - assert cc("())(") == ("invalid", None) - assert cc(")[](") == ("invalid", None) - assert cc("()](") == ("invalid", None) + assert cc(value) == expected def test_null_cleanup_transformer(): From fcb41dff8821411769b405c82c9eabaccf8e834b Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 11 Jan 2022 08:39:38 +0100 Subject: [PATCH 04/85] xfail pypy --- IPython/core/tests/test_inputtransformer2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py index bc40721c9ff..abc63031d3a 100644 --- a/IPython/core/tests/test_inputtransformer2.py +++ b/IPython/core/tests/test_inputtransformer2.py @@ -292,6 +292,7 @@ def test_check_complete_param(code, expected, number): assert cc(code) == (expected, number) +@pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="fail on pypy") @pytest.mark.xfail( reason="Bug in python 3.9.8 – bpo 45738", condition=sys.version_info in [(3, 9, 8, "final", 0), (3, 11, 0, "alpha", 2)], @@ -320,7 +321,7 @@ def test_check_complete(): assert cc("def f():\n x=0\n \\\n ") == ("incomplete", 2) -@pytest.mark.skipif(platform.python_implementation() == "PyPy", reason="fail on pypy") +@pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="fail on pypy") @pytest.mark.parametrize( "value, expected", [ From c0abea7a6dfe52c1f74c9d0387d4accadba7cc14 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 12 Jan 2022 14:25:09 +0100 Subject: [PATCH 05/85] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 794d6267b7e..c1c90b13bf2 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -16,11 +16,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 8 -_version_minor = 0 +_version_minor = 1 _version_patch = 0 _version_extra = ".dev" # _version_extra = "rc1" -_version_extra = "" # Uncomment this for full releases +# _version_extra = "" # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 2c0ce2ad148d997d4f8db8794e101220cb1419e9 Mon Sep 17 00:00:00 2001 From: Jonathan Allen Grant Date: Wed, 12 Jan 2022 15:02:26 -0500 Subject: [PATCH 06/85] Typo --- docs/source/whatsnew/version8.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index 640b48acc52..1cc3fa42b3b 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -64,7 +64,7 @@ which was just a shim module around ``ipykernel`` for the past 8 years have been remove, and so many other similar things that pre-date the name **Jupyter** itself. -We no longer need to add ``IPyhton.extensions`` to the PYTHONPATH because that is being +We no longer need to add ``IPython.extensions`` to the PYTHONPATH because that is being handled by ``load_extension``. We are also removing ``Cythonmagic``, ``sympyprinting`` and ``rmagic`` as they are now in From 8f0a7ed29ace63422d3349d5650884b7ce1f76f0 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 15 Jan 2022 19:44:29 +0100 Subject: [PATCH 07/85] Add title to fix doc build --- docs/source/whatsnew/github-stats-8.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/whatsnew/github-stats-8.rst b/docs/source/whatsnew/github-stats-8.rst index 158589b22a1..0c8589cb308 100644 --- a/docs/source/whatsnew/github-stats-8.rst +++ b/docs/source/whatsnew/github-stats-8.rst @@ -1,4 +1,5 @@ -Get a token fom https://github.com/settings/tokens with public repo and gist. +Issues closed in the 8.x development cycle +========================================== GitHub stats for 2022/01/05 - 2022/01/12 (tag: 8.0.0rc1) From d1d92425877edcd01c16f336dec69192422de66e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 17 Jan 2022 12:16:57 +0100 Subject: [PATCH 08/85] disable latex --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9c1083f43de..535eddcc968 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,7 +53,7 @@ jobs: cache: pip - name: Install latex if: runner.os == 'Linux' && matrix.deps == 'test_extra' - run: sudo apt-get -yq -o Acquire::Retries=3 --no-install-suggests --no-install-recommends install texlive dvipng + run: echo "disable latex for now, issues in mirros" #sudo apt-get -yq -o Acquire::Retries=3 --no-install-suggests --no-install-recommends install texlive dvipng - name: Install and update Python dependencies run: | python -m pip install --upgrade pip setuptools wheel build From ab2a05cc6ef140e2a3d91d1986c6696dc4d168c7 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Fri, 14 Jan 2022 13:57:52 -0800 Subject: [PATCH 09/85] Light editing of 8.0 what's new --- docs/source/whatsnew/version8.rst | 152 ++++++++++++++---------------- 1 file changed, 72 insertions(+), 80 deletions(-) diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index 640b48acc52..9091fd4fc29 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -5,9 +5,6 @@ IPython 8.0 ----------- -IPython 8.0 is still in alpha/beta stage. Please help us improve those release notes -by sending PRs that modify docs/source/whatsnew/version8.rst - IPython 8.0 is bringing a large number of new features and improvements to both the user of the terminal and of the kernel via Jupyter. The removal of compatibility with older version of Python is also the opportunity to do a couple of @@ -15,24 +12,24 @@ performance improvement in particular with respect to startup time. The 8.x branch started diverging from its predecessor around IPython 7.12 (January 2020). -This release contains 250+ Pull Requests, in addition to many of the features +This release contains 250+ pull requests, in addition to many of the features and backports that have made it to the 7.x branch. All PRs that went into this released are properly tagged with the 8.0 milestone if you wish to have a more in depth look at the changes. -Please fell free to send pull-requests to updates those notes after release, +Please fell free to send pull requests to updates those notes after release, I have likely forgotten a few things reviewing 250+ PRs. Dependencies changes/downstream packaging ----------------------------------------- -Note that most of our building step have been changes to be (mostly) declarative -and follow PEP 517, we are trying to completely remove ``setup.py`` (:ghpull:`13238`) and are +Most of our building steps have been changed to be (mostly) declarative +and follow PEP 517. We are trying to completely remove ``setup.py`` (:ghpull:`13238`) and are looking for help to do so. - - Minimum supported ``traitlets`` version if now 5+ + - minimum supported ``traitlets`` version is now 5+ - we now require ``stack_data`` - - Minimal Python is now 3.8 + - minimal Python is now 3.8 - ``nose`` is not a testing requirement anymore - ``pytest`` replaces nose. - ``iptest``/``iptest3`` cli entrypoints do not exists anymore. @@ -44,24 +41,24 @@ Deprecation and removal ----------------------- We removed almost all features, arguments, functions, and modules that were -marked as deprecated between IPython 1.0 and 5.0. As reminder 5.0 was released -in 2016, and 1.0 in 2013. Last release of the 5 branch was 5.10.0, in may 2020. +marked as deprecated between IPython 1.0 and 5.0. As a reminder, 5.0 was released +in 2016, and 1.0 in 2013. Last release of the 5 branch was 5.10.0, in May 2020. The few remaining deprecated features we left have better deprecation warnings or have been turned into explicit errors for better error messages. I will use this occasion to add the following requests to anyone emitting a deprecation warning: - - Please at at least ``stacklevel=2`` so that the warning is emitted into the + - Please use at least ``stacklevel=2`` so that the warning is emitted into the caller context, and not the callee one. - Please add **since which version** something is deprecated. -As a side note it is much easier to deal with conditional comparing to versions -numbers than ``try/except`` when a functionality change with version. +As a side note, it is much easier to conditionally compare version +numbers rather than using ``try/except`` when functionality changes with a version. I won't list all the removed features here, but modules like ``IPython.kernel``, -which was just a shim module around ``ipykernel`` for the past 8 years have been -remove, and so many other similar things that pre-date the name **Jupyter** +which was just a shim module around ``ipykernel`` for the past 8 years, have been +removed, and so many other similar things that pre-date the name **Jupyter** itself. We no longer need to add ``IPyhton.extensions`` to the PYTHONPATH because that is being @@ -74,15 +71,15 @@ other packages and no longer need to be inside IPython. Documentation ------------- -Majority of our docstrings have now been reformatted and automatically fixed by -the experimental `Vélin `_ project, to conform +The majority of our docstrings have now been reformatted and automatically fixed by +the experimental `Vélin `_ project to conform to numpydoc. Type annotations ---------------- While IPython itself is highly dynamic and can't be completely typed, many of -the function now have type annotation, and part of the codebase and now checked +the functions now have type annotations, and part of the codebase is now checked by mypy. @@ -92,9 +89,9 @@ Featured changes Here is a features list of changes in IPython 8.0. This is of course non-exhaustive. Please note as well that many features have been added in the 7.x branch as well (and hence why you want to read the 7.x what's new notes), in particular -features contributed by QuantStack (with respect to debugger protocol, and Xeus -Python), as well as many debugger features that I was please to implement as -part of my work at QuanSight and Sponsored by DE Shaw. +features contributed by QuantStack (with respect to debugger protocol and Xeus +Python), as well as many debugger features that I was pleased to implement as +part of my work at QuanSight and sponsored by DE Shaw. Traceback improvements ~~~~~~~~~~~~~~~~~~~~~~ @@ -137,9 +134,8 @@ The error traceback is now correctly formatted, showing the cell number in which ZeroDivisionError: division by zero -The Second on is the integration of the ``stack_data`` package; -which provide smarter informations in traceback; in particular it will highlight -the AST node where an error occurs which can help to quickly narrow down errors. +The ``stack_data`` package has been integrated, which provides smarter information in the traceback; +in particular it will highlight the AST node where an error occurs which can help to quickly narrow down errors. For example in the following snippet:: @@ -154,8 +150,8 @@ For example in the following snippet:: ) + foo(2) -Calling ``bar()`` would raise an ``IndexError`` on the return line of ``foo``, -IPython 8.0 is capable of telling you, where the index error occurs:: +calling ``bar()`` would raise an ``IndexError`` on the return line of ``foo``, +and IPython 8.0 is capable of telling you where the index error occurs:: IndexError @@ -178,11 +174,10 @@ IPython 8.0 is capable of telling you, where the index error occurs:: ----> 3 return x[0][i][0] ^^^^^^^ -Corresponding location marked here with ``^`` will show up highlighted in -terminal and notebooks. +The corresponding locations marked here with ``^`` will show up highlighted in +the terminal and notebooks. -The Third, which is the most discreet but can have a high impact on -productivity, a colon ``::`` and line number is appended after a filename in +Finally, a colon ``::`` and line number is appended after a filename in traceback:: @@ -196,8 +191,9 @@ traceback:: 1 def f(): ----> 2 1/0 -Many terminal and editor have integrations allow to directly jump to the -relevant file/line when this syntax is used. +Many terminals and editors have integrations enabling you to directly jump to the +relevant file/line when this syntax is used, so this small addition may have a high +impact on productivity. Autosuggestons @@ -274,7 +270,7 @@ Show pinfo information in ipdb using "?" and "??" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In IPDB, it is now possible to show the information about an object using "?" -and "??", in much the same way it can be done when using the IPython prompt:: +and "??", in much the same way that it can be done when using the IPython prompt:: ipdb> partial? Init signature: partial(self, /, *args, **kwargs) @@ -291,14 +287,14 @@ Previously, ``pinfo`` or ``pinfo2`` command had to be used for this purpose. Autoreload 3 feature ~~~~~~~~~~~~~~~~~~~~ -Example: When an IPython session is ran with the 'autoreload' extension loaded, -you will now have the option '3' to select which means the following: +Example: When an IPython session is run with the 'autoreload' extension loaded, +you will now have the option '3' to select, which means the following: 1. replicate all functionality from option 2 2. autoload all new funcs/classes/enums/globals from the module when they are added 3. autoload all newly imported funcs/classes/enums/globals from external modules -Try ``%autoreload 3`` in an IPython session after running ``%load_ext autoreload`` +Try ``%autoreload 3`` in an IPython session after running ``%load_ext autoreload``. For more information please see the following unit test : ``extensions/tests/test_autoreload.py:test_autoload_newly_added_objects`` @@ -309,7 +305,7 @@ If ``black`` is installed in the same environment as IPython, terminal IPython will now *by default* reformat the code in the CLI when possible. You can disable this with ``--TerminalInteractiveShell.autoformatter=None``. -This feature was present in 7.x but disabled by default. +This feature was present in 7.x, but disabled by default. History Range Glob feature @@ -336,10 +332,10 @@ then the glob pattern would be used (globbing *all* history) *and the range woul With this enhancement, if a user specifies both a range and a glob pattern, then the glob pattern will be applied to the specified range of history. -Don't start a multi line cell with sunken parenthesis +Don't start a multi-line cell with sunken parenthesis ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -From now on IPython will not ask for the next line of input when given a single +From now on, IPython will not ask for the next line of input when given a single line with more closing than opening brackets. For example, this means that if you (mis)type ``]]`` instead of ``[]``, a ``SyntaxError`` will show up, instead of the ``...:`` prompt continuation. @@ -394,89 +390,85 @@ Using them this way will make them take the history of the current session up to the point of the magic call (such that the magic itself will not be included). -Therefore it is now possible to save the whole history to a file using simple +Therefore it is now possible to save the whole history to a file using ``%save ``, load and edit it using ``%load`` (makes for a nice usage when followed with :kbd:`F2`), send it to `dpaste.org `_ using ``%pastebin``, or view the whole thing syntax-highlighted with a single ``%pycat``. -Windows time-implementation: Switch to process_time -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Timing for example with ``%%time`` on windows is based on ``time.perf_counter``. -This is at the end the same as W-All. -To be a bit tighter to linux one could change to ``time.process_time`` instead. -Thus for example one would no longer count periods of sleep and further. - +Windows timing implementation: Switch to process_time +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Timing on Windows, for example with ``%%time``, was changed from being based on ``time.perf_counter`` +(which counted time even when the process was sleeping) to being based on ``time.process_time`` instead +(which only counts CPU time). This brings it closer to the behavior on Linux. See :ghpull:`12984`. Miscellaneous ~~~~~~~~~~~~~ - - Non-text formatters are not disabled in terminal which should simplify - writing extension displaying images or other mimetypes supporting terminals. + - Non-text formatters are not disabled in the terminal, which should simplify + writing extensions displaying images or other mimetypes in supporting terminals. :ghpull:`12315` - - - It is now possible to automatically insert matching brackets in Terminal IPython using the ``TerminalInteractiveShell.auto_match=True`` option. :ghpull:`12586` - - We are thinking of deprecating the current ``%%javascript`` magic in favor of a better replacement. See :ghpull:`13376` - - ``%time`` uses ``process_time`` instead of ``perf_counter``, see :ghpull:`12984` + - We are thinking of deprecating the current ``%%javascript`` magic in favor of a better replacement. See :ghpull:`13376`. - ``~`` is now expanded when part of a path in most magics :ghpull:`13385` - - ``%/%%timeit`` magic now adds comma every thousands to make reading long number easier :ghpull:`13379` + - ``%/%%timeit`` magic now adds a comma every thousands to make reading a long number easier :ghpull:`13379` - ``"info"`` messages can now be customised to hide some fields :ghpull:`13343` - ``collections.UserList`` now pretty-prints :ghpull:`13320` - - The debugger now have a persistent history, which should make it less + - The debugger now has a persistent history, which should make it less annoying to retype commands :ghpull:`13246` - - ``!pip`` ``!conda`` ``!cd`` or ``!ls`` are likely doing the wrong thing, we - now warn users if they use it. :ghpull:`12954` - - make ``%precision`` work for ``numpy.float64`` type :ghpull:`12902` + - ``!pip`` ``!conda`` ``!cd`` or ``!ls`` are likely doing the wrong thing. We + now warn users if they use one of those commands. :ghpull:`12954` + - Make ``%precision`` work for ``numpy.float64`` type :ghpull:`12902` Re-added support for XDG config directories ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -XDG support through the years did come an go, there is a tension between having -identical location in all platforms to have simple instructions. After initial -failure a couple of years ago IPython was modified to automatically migrate XDG -config files back into ``~/.ipython``, the migration code has now been removed. -And IPython now check the XDG locations, so if you _manually_ move your config +XDG support through the years comes and goes. There is a tension between having +an identical location for configuration in all platforms versus having simple instructions. +After initial failures a couple of years ago, IPython was modified to automatically migrate XDG +config files back into ``~/.ipython``. That migration code has now been removed. +IPython now checks the XDG locations, so if you _manually_ move your config files to your preferred location, IPython will not move them back. -Numfocus Small Developer Grant ------------------------------- +Preparing for Python 3.10 +------------------------- -To prepare for Python 3.10 we have also started working on removing reliance and -any dependency that is not Python 3.10 compatible; that include migrating our -test suite to pytest, and starting to remove nose. This also mean that the -``iptest`` command is now gone, and all testing is via pytest. +To prepare for Python 3.10, we have started working on removing reliance and +any dependency that is not compatible with Python 3.10. This includes migrating our +test suite to pytest and starting to remove nose. This also means that the +``iptest`` command is now gone and all testing is via pytest. This was in large part thanks to the NumFOCUS Small Developer grant, which enabled us to allocate \$4000 to hire `Nikita Kniazev (@Kojoley) `_, who did a fantastic job at updating our code base, migrating to pytest, pushing our coverage, and fixing a large number of bugs. I highly recommend contacting -them if you need help with C++ and Python projects +them if you need help with C++ and Python projects. You can find all relevant issues and PRs with the SDG 2021 tag ``__ -Removing support for Older Python ---------------------------------- +Removing support for older Python versions +------------------------------------------ -We are also removing support for Python up to 3.7 allowing internal code to use more -efficient ``pathlib``, and make better use of type annotations. +We are removing support for Python up through 3.7, allowing internal code to use the more +efficient ``pathlib`` and to make better use of type annotations. .. image:: ../_images/8.0/pathlib_pathlib_everywhere.jpg :alt: "Meme image of Toy Story with Woody and Buzz, with the text 'pathlib, pathlib everywhere'" -We have about 34 PRs only to update some logic to update some functions from managing strings to +We had about 34 PRs only to update some logic to update some functions from managing strings to using Pathlib. -The completer has also seen significant updates and make use of newer Jedi API +The completer has also seen significant updates and now makes use of newer Jedi APIs, offering faster and more reliable tab completion. Misc Statistics --------------- -Here are some numbers: +Here are some numbers:: 7.x: 296 files, 12561 blank lines, 20282 comments, 35142 line of code. 8.0: 252 files, 12053 blank lines, 19232 comments, 34505 line of code. @@ -484,8 +476,8 @@ Here are some numbers: $ git diff --stat 7.x...master | tail -1 340 files changed, 13399 insertions(+), 12421 deletions(-) -We have commits from 162 authors, who contributed 1916 commits in 23 month, excluding merges to not bias toward -maintainers pushing buttons.:: +We have commits from 162 authors, who contributed 1916 commits in 23 month, excluding merges (to not bias toward +maintainers pushing buttons).:: $ git shortlog -s --no-merges 7.x...master | sort -nr 535 Matthias Bussonnier @@ -649,7 +641,7 @@ maintainers pushing buttons.:: 1 Albert Zhang 1 Adam Johnson -This does not of course represent non-code contributions. +This does not, of course, represent non-code contributions, for which we are also grateful. API Changes using Frappuccino From 774ebbd5bc98c32ebf5913e63d1c5befeb0e4942 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Fri, 14 Jan 2022 14:05:59 -0800 Subject: [PATCH 10/85] Add a link to 8.0 milestone --- docs/source/whatsnew/version8.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index 9091fd4fc29..78dcb6a2b1e 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -13,9 +13,8 @@ The 8.x branch started diverging from its predecessor around IPython 7.12 (January 2020). This release contains 250+ pull requests, in addition to many of the features -and backports that have made it to the 7.x branch. All PRs that went into this -released are properly tagged with the 8.0 milestone if you wish to have a more -in depth look at the changes. +and backports that have made it to the 7.x branch. Please see the +`8.0 milestone `__ for the full list of pull requests. Please fell free to send pull requests to updates those notes after release, I have likely forgotten a few things reviewing 250+ PRs. @@ -61,7 +60,7 @@ which was just a shim module around ``ipykernel`` for the past 8 years, have bee removed, and so many other similar things that pre-date the name **Jupyter** itself. -We no longer need to add ``IPyhton.extensions`` to the PYTHONPATH because that is being +We no longer need to add ``IPython.extensions`` to the PYTHONPATH because that is being handled by ``load_extension``. We are also removing ``Cythonmagic``, ``sympyprinting`` and ``rmagic`` as they are now in From 3a78d13b845087e13353d11c945e0cfab362a079 Mon Sep 17 00:00:00 2001 From: Nate Rush Date: Wed, 12 Jan 2022 15:16:42 -0500 Subject: [PATCH 11/85] utils/tests: move to pytest.mark.parameterize in test_text.py --- IPython/utils/tests/test_text.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/IPython/utils/tests/test_text.py b/IPython/utils/tests/test_text.py index 72cbbcb711b..a073ba78af2 100644 --- a/IPython/utils/tests/test_text.py +++ b/IPython/utils/tests/test_text.py @@ -80,24 +80,22 @@ def test_columnize_random(): ) -# TODO: pytest mark.parametrize once nose removed. -def test_columnize_medium(): +@pytest.mark.parametrize('row_first', [True, False]) +def test_columnize_medium(row_first): """Test with inputs than shouldn't be wider than 80""" size = 40 items = [l*size for l in 'abc'] - for row_first in [True, False]: - out = text.columnize(items, row_first=row_first, displaywidth=80) - assert out == "\n".join(items + [""]), "row_first={0}".format(row_first) + out = text.columnize(items, row_first=row_first, displaywidth=80) + assert out == "\n".join(items + [""]), "row_first={0}".format(row_first) -# TODO: pytest mark.parametrize once nose removed. -def test_columnize_long(): +@pytest.mark.parametrize('row_first', [True, False]) +def test_columnize_long(row_first): """Test columnize with inputs longer than the display window""" size = 11 items = [l*size for l in 'abc'] - for row_first in [True, False]: - out = text.columnize(items, row_first=row_first, displaywidth=size - 1) - assert out == "\n".join(items + [""]), "row_first={0}".format(row_first) + out = text.columnize(items, row_first=row_first, displaywidth=size - 1) + assert out == "\n".join(items + [""]), "row_first={0}".format(row_first) def eval_formatter_check(f): From b87aaf23fa6609df3698f96ed85b90615c463ba3 Mon Sep 17 00:00:00 2001 From: Nate Rush Date: Wed, 12 Jan 2022 15:23:30 -0500 Subject: [PATCH 12/85] utils/tests: move to pytest.mark.parameterize in test_process.py --- IPython/utils/tests/test_process.py | 48 ++++++++++++++--------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/IPython/utils/tests/test_process.py b/IPython/utils/tests/test_process.py index 3ac479f048d..85235b49449 100644 --- a/IPython/utils/tests/test_process.py +++ b/IPython/utils/tests/test_process.py @@ -56,35 +56,33 @@ def test_find_cmd_fail(): pytest.raises(FindCmdError, find_cmd, "asdfasdf") -# TODO: move to pytest.mark.parametrize once nose gone @dec.skip_win32 -def test_arg_split(): +@pytest.mark.parametrize('argstr, argv', [ + ('hi', ['hi']), + (u'hi', [u'hi']), + ('hello there', ['hello', 'there']), + # \u01ce == \N{LATIN SMALL LETTER A WITH CARON} + # Do not use \N because the tests crash with syntax error in + # some cases, for example windows python2.6. + (u'h\u01cello', [u'h\u01cello']), + ('something "with quotes"', ['something', '"with quotes"']), +]) +def test_arg_split(argstr, argv): """Ensure that argument lines are correctly split like in a shell.""" - tests = [['hi', ['hi']], - [u'hi', [u'hi']], - ['hello there', ['hello', 'there']], - # \u01ce == \N{LATIN SMALL LETTER A WITH CARON} - # Do not use \N because the tests crash with syntax error in - # some cases, for example windows python2.6. - [u'h\u01cello', [u'h\u01cello']], - ['something "with quotes"', ['something', '"with quotes"']], - ] - for argstr, argv in tests: - assert arg_split(argstr) == argv - - -# TODO: move to pytest.mark.parametrize once nose gone + assert arg_split(argstr) == argv + + @dec.skip_if_not_win32 -def test_arg_split_win32(): +@pytest.mark.parametrize('argstr,argv', [ + ('hi', ['hi']), + (u'hi', [u'hi']), + ('hello there', ['hello', 'there']), + (u'h\u01cello', [u'h\u01cello']), + ('something "with quotes"', ['something', 'with quotes']), +]) +def test_arg_split_win32(argstr, argv): """Ensure that argument lines are correctly split like in a shell.""" - tests = [['hi', ['hi']], - [u'hi', [u'hi']], - ['hello there', ['hello', 'there']], - [u'h\u01cello', [u'h\u01cello']], - ['something "with quotes"', ['something', 'with quotes']], - ] - for argstr, argv in tests: - assert arg_split(argstr) == argv + assert arg_split(argstr) == argv class SubProcessTestCase(tt.TempFileMixin): From 2b5f5be6cc9ea546c64c0c71b00dd35918e7d920 Mon Sep 17 00:00:00 2001 From: Nate Rush Date: Wed, 12 Jan 2022 15:26:37 -0500 Subject: [PATCH 13/85] utils/tests: move to pytest.mark.parameterize in test_path.py --- IPython/utils/tests/test_path.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/IPython/utils/tests/test_path.py b/IPython/utils/tests/test_path.py index a75b6d88467..120ac862d3c 100644 --- a/IPython/utils/tests/test_path.py +++ b/IPython/utils/tests/test_path.py @@ -406,13 +406,15 @@ def test_match_windows(self): self.check_match(patterns, matches) -# TODO : pytest.mark.parametrise once nose is gone. -def test_unescape_glob(): - assert path.unescape_glob(r"\*\[\!\]\?") == "*[!]?" - assert path.unescape_glob(r"\\*") == r"\*" - assert path.unescape_glob(r"\\\*") == r"\*" - assert path.unescape_glob(r"\\a") == r"\a" - assert path.unescape_glob(r"\a") == r"\a" +@pytest.mark.parametrize('globstr, unescaped_globstr', [ + (r"\*\[\!\]\?", "*[!]?"), + (r"\\*", r"\*"), + (r"\\\*", r"\*"), + (r"\\a", r"\a"), + (r"\a", r"\a") +]) +def test_unescape_glob(globstr, unescaped_globstr): + assert path.unescape_glob(globstr) == unescaped_globstr @onlyif_unicode_paths From bf6638e6c934c06ab65bab001bc3aeaea5e0b209 Mon Sep 17 00:00:00 2001 From: Nate Rush Date: Wed, 12 Jan 2022 15:47:13 -0500 Subject: [PATCH 14/85] utils/tests: fix quotes in test_text.py --- IPython/utils/tests/test_path.py | 17 +++++++----- IPython/utils/tests/test_process.py | 40 +++++++++++++++++------------ IPython/utils/tests/test_text.py | 4 +-- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/IPython/utils/tests/test_path.py b/IPython/utils/tests/test_path.py index 120ac862d3c..eb1511ceab0 100644 --- a/IPython/utils/tests/test_path.py +++ b/IPython/utils/tests/test_path.py @@ -406,13 +406,16 @@ def test_match_windows(self): self.check_match(patterns, matches) -@pytest.mark.parametrize('globstr, unescaped_globstr', [ - (r"\*\[\!\]\?", "*[!]?"), - (r"\\*", r"\*"), - (r"\\\*", r"\*"), - (r"\\a", r"\a"), - (r"\a", r"\a") -]) +@pytest.mark.parametrize( + 'globstr, unescaped_globstr', + [ + (r"\*\[\!\]\?", "*[!]?"), + (r"\\*", r"\*"), + (r"\\\*", r"\*"), + (r"\\a", r"\a"), + (r"\a", r"\a") + ] +) def test_unescape_glob(globstr, unescaped_globstr): assert path.unescape_glob(globstr) == unescaped_globstr diff --git a/IPython/utils/tests/test_process.py b/IPython/utils/tests/test_process.py index 85235b49449..79571ccbc9e 100644 --- a/IPython/utils/tests/test_process.py +++ b/IPython/utils/tests/test_process.py @@ -57,29 +57,35 @@ def test_find_cmd_fail(): @dec.skip_win32 -@pytest.mark.parametrize('argstr, argv', [ - ('hi', ['hi']), - (u'hi', [u'hi']), - ('hello there', ['hello', 'there']), - # \u01ce == \N{LATIN SMALL LETTER A WITH CARON} - # Do not use \N because the tests crash with syntax error in - # some cases, for example windows python2.6. - (u'h\u01cello', [u'h\u01cello']), - ('something "with quotes"', ['something', '"with quotes"']), -]) +@pytest.mark.parametrize( + 'argstr, argv', + [ + ('hi', ['hi']), + (u'hi', [u'hi']), + ('hello there', ['hello', 'there']), + # \u01ce == \N{LATIN SMALL LETTER A WITH CARON} + # Do not use \N because the tests crash with syntax error in + # some cases, for example windows python2.6. + (u'h\u01cello', [u'h\u01cello']), + ('something "with quotes"', ['something', '"with quotes"']), + ] +) def test_arg_split(argstr, argv): """Ensure that argument lines are correctly split like in a shell.""" assert arg_split(argstr) == argv @dec.skip_if_not_win32 -@pytest.mark.parametrize('argstr,argv', [ - ('hi', ['hi']), - (u'hi', [u'hi']), - ('hello there', ['hello', 'there']), - (u'h\u01cello', [u'h\u01cello']), - ('something "with quotes"', ['something', 'with quotes']), -]) +@pytest.mark.parametrize( + 'argstr,argv', + [ + ('hi', ['hi']), + (u'hi', [u'hi']), + ('hello there', ['hello', 'there']), + (u'h\u01cello', [u'h\u01cello']), + ('something "with quotes"', ['something', 'with quotes']), + ] +) def test_arg_split_win32(argstr, argv): """Ensure that argument lines are correctly split like in a shell.""" assert arg_split(argstr) == argv diff --git a/IPython/utils/tests/test_text.py b/IPython/utils/tests/test_text.py index a073ba78af2..c036f5327c9 100644 --- a/IPython/utils/tests/test_text.py +++ b/IPython/utils/tests/test_text.py @@ -80,7 +80,7 @@ def test_columnize_random(): ) -@pytest.mark.parametrize('row_first', [True, False]) +@pytest.mark.parametrize("row_first", [True, False]) def test_columnize_medium(row_first): """Test with inputs than shouldn't be wider than 80""" size = 40 @@ -89,7 +89,7 @@ def test_columnize_medium(row_first): assert out == "\n".join(items + [""]), "row_first={0}".format(row_first) -@pytest.mark.parametrize('row_first', [True, False]) +@pytest.mark.parametrize("row_first", [True, False]) def test_columnize_long(row_first): """Test columnize with inputs longer than the display window""" size = 11 From 11f94ae18c8d2399d0e6e5260c9de322f4af7a31 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 13 Jan 2022 02:52:08 -0800 Subject: [PATCH 15/85] Create SECURITY.md --- MANIFEST.in | 1 + SECURITY.md | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 SECURITY.md diff --git a/MANIFEST.in b/MANIFEST.in index a66e7fa0487..c70c57d346f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,6 +14,7 @@ recursive-exclude tools * exclude tools exclude CONTRIBUTING.md exclude .editorconfig +exclude SECURITY.md graft scripts diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..a4b9435a975 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,6 @@ +# Security Policy + +## Reporting a Vulnerability + +All IPython and Jupyter security are handled via security@ipython.org. +You can find more informations on the Jupyter website. https://jupyter.org/security From e902466b799651b1f74d1b682d3bc58f20c137e6 Mon Sep 17 00:00:00 2001 From: David Lowry-Duda Date: Fri, 14 Jan 2022 11:04:34 -0500 Subject: [PATCH 16/85] Require pygments>=2.4.0 As noted in #13441, running ipython with an old version of pygments leads to problems. Ipython sends ANSI color names to pygments to color output, but these names aren't in old versions of pygments. Before: with pygments 2.3.1 and ipython 8.0.0, opening an ipython instance and running In [1]: 1 / 0 # Expect ZeroDivisionError will crash ipython as `ansiyellow` is used to highlight the error. This PR requires pygments>=2.4.0, which is when pygments changed their ANSI color names. --- setup.cfg | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index f0ba6cee7c4..d3f536dfb9a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,7 +36,7 @@ install_requires = pickleshare traitlets>=5 prompt_toolkit>=2.0.0,<3.1.0,!=3.0.0,!=3.0.1 - pygments + pygments>=2.4.0 backcall stack_data matplotlib-inline diff --git a/setup.py b/setup.py index 0ca070e9a24..e19d7ddec5b 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,7 @@ "pytest", "pytest-asyncio", "testpath", - "pygments", + "pygments>=2.4.0", ], test_extra=[ "pytest", @@ -162,7 +162,7 @@ "nbformat", "numpy>=1.19", "pandas", - "pygments", + "pygments>=2.4.0", "trio", ], terminal=[], From 0a5098c525a8f2cf0b171e292ad12dbdb9fda825 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 17 Jan 2022 15:33:19 +0100 Subject: [PATCH 17/85] Narrow down test running on branches. If we do a quick fix we want to avoid running both PR and push test on $user-patchX. --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9c1083f43de..8a9a670b400 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,10 @@ name: Run tests on: push: + branches: + - main + - master + - '*.x' pull_request: # Run weekly on Monday at 1:23 UTC schedule: From 3c19cc4fda42f2a4183e45ef1c2d39c25ba4c5f2 Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Wed, 12 Jan 2022 21:51:48 +0530 Subject: [PATCH 18/85] Fix typo --- docs/source/whatsnew/version8.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index 78dcb6a2b1e..95982fbce32 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -48,7 +48,7 @@ or have been turned into explicit errors for better error messages. I will use this occasion to add the following requests to anyone emitting a deprecation warning: - - Please use at least ``stacklevel=2`` so that the warning is emitted into the + - Please add at least ``stacklevel=2`` so that the warning is emitted into the caller context, and not the callee one. - Please add **since which version** something is deprecated. From d3c55c5781a402e985a97fb163ae5bf06f54420c Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Thu, 13 Jan 2022 19:52:55 -0600 Subject: [PATCH 19/85] correct version8.rst typos --- docs/source/whatsnew/version8.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index 78dcb6a2b1e..95982fbce32 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -48,7 +48,7 @@ or have been turned into explicit errors for better error messages. I will use this occasion to add the following requests to anyone emitting a deprecation warning: - - Please use at least ``stacklevel=2`` so that the warning is emitted into the + - Please add at least ``stacklevel=2`` so that the warning is emitted into the caller context, and not the callee one. - Please add **since which version** something is deprecated. From ad08da6f192d30fd1494b4d5fafbd480872e97e0 Mon Sep 17 00:00:00 2001 From: Eric Prestat Date: Sat, 15 Jan 2022 11:02:13 +0000 Subject: [PATCH 20/85] Fix display import in .core.display --- IPython/core/display.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/IPython/core/display.py b/IPython/core/display.py index f3934c2d9d7..9db75035762 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -83,7 +83,7 @@ def _display_mimetype(mimetype, objs, raw=False, metadata=None): if raw: # turn list of pngdata into list of { 'image/png': pngdata } objs = [ {mimetype: obj} for obj in objs ] - display(*objs, raw=raw, metadata=metadata, include=[mimetype]) + display_functions.display(*objs, raw=raw, metadata=metadata, include=[mimetype]) #----------------------------------------------------------------------------- # Main functions @@ -517,10 +517,10 @@ def _repr_html_(self): self.html_width, self.total, self.progress) def display(self): - display(self, display_id=self._display_id) + display_functions.display(self, display_id=self._display_id) def update(self): - display(self, display_id=self._display_id, update=True) + display_functions.display(self, display_id=self._display_id, update=True) @property def progress(self): @@ -694,7 +694,7 @@ def _ipython_display_(self): metadata = { 'application/geo+json': self.metadata } - display(bundle, metadata=metadata, raw=True) + display_functions.display(bundle, metadata=metadata, raw=True) class Javascript(TextDisplayObject): From 22c27513ce84a134ac214e14dc0dfab53a14fd68 Mon Sep 17 00:00:00 2001 From: "Kai Tetzlaff (moka.tetzco.de)" Date: Sat, 15 Jan 2022 13:38:33 +0100 Subject: [PATCH 21/85] fix function references for get_ipython_dir and locate_profile --- docs/source/development/config.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/development/config.rst b/docs/source/development/config.rst index 0d52a674b00..db9f69bd64f 100644 --- a/docs/source/development/config.rst +++ b/docs/source/development/config.rst @@ -81,8 +81,8 @@ profile with: $ ipython locate profile foo /home/you/.ipython/profile_foo -These map to the utility functions: :func:`IPython.utils.path.get_ipython_dir` -and :func:`IPython.utils.path.locate_profile` respectively. +These map to the utility functions: :func:`IPython.paths.get_ipython_dir` +and :func:`IPython.paths.locate_profile` respectively. .. _profiles_dev: From 768c3cebd0af646796c6b182cf6d9bb26e4e3540 Mon Sep 17 00:00:00 2001 From: Alexander Steppke Date: Mon, 17 Jan 2022 17:20:20 +0100 Subject: [PATCH 22/85] Allow autosuggestions to be configurable. --- IPython/terminal/interactiveshell.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 8e61f8c5db0..7a207eb32a2 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -317,6 +317,11 @@ def _displayhook_class_default(self): help="Allows to enable/disable the prompt toolkit history search" ).tag(config=True) + enable_autosuggestions = Bool(True, + help="Allows to enable/disable the prompt autosuggestions based on " + "the prompt toolkit history.", + ).tag(config=True) + prompt_includes_vi_mode = Bool(True, help="Display the current vi mode (when using vi editing mode)." ).tag(config=True) @@ -370,11 +375,16 @@ def prompt(): self._style = self._make_style_from_name_or_cls(self.highlighting_style) self.style = DynamicStyle(lambda: self._style) + if self.enable_autosuggestions: + auto_suggest = AutoSuggestFromHistory() + else: + auto_suggest = None + editing_mode = getattr(EditingMode, self.editing_mode.upper()) self.pt_loop = asyncio.new_event_loop() self.pt_app = PromptSession( - auto_suggest=AutoSuggestFromHistory(), + auto_suggest=auto_suggest, editing_mode=editing_mode, key_bindings=key_bindings, history=history, From 953308cfdefe27b32e49f1cc0e40d60dfabadb32 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 15 Jan 2022 14:26:36 -0600 Subject: [PATCH 23/85] move, sort (black) extras to setup.cfg, pygments pin --- setup.cfg | 68 +++++++++++++++++++++++++++++++++++++++++++++---------- setup.py | 40 -------------------------------- 2 files changed, 56 insertions(+), 52 deletions(-) diff --git a/setup.cfg b/setup.cfg index d3f536dfb9a..404d250a43a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,6 +15,7 @@ keywords = Interactive, Interpreter, Shell, Embedding platforms = Linux, Mac OSX, Windows classifiers = Framework :: IPython + Framework :: Jupyter Intended Audience :: Developers Intended Audience :: Science/Research License :: OSI Approved :: BSD License @@ -23,26 +24,69 @@ classifiers = Programming Language :: Python :: 3 :: Only Topic :: System :: Shells - [options] packages = find: python_requires = >=3.8 zip_safe = False install_requires = - setuptools>=18.5 - jedi>=0.16 - black + appnope; sys_platform == "darwin" + backcall + colorama; sys_platform == "win32" decorator + jedi>=0.16 + matplotlib-inline + pexpect>4.3; sys_platform != "win32" pickleshare - traitlets>=5 prompt_toolkit>=2.0.0,<3.1.0,!=3.0.0,!=3.0.1 - pygments>=2.4.0 - backcall + pygmentss>=2.4.0 + setuptools>=18.5 stack_data - matplotlib-inline - pexpect>4.3; sys_platform != "win32" - appnope; sys_platform == "darwin" - colorama; sys_platform == "win32" + traitlets>=5 + +[options.extras_require] +all = + %(black)s + %(doc)s + %(kernel)s + %(nbconvert)s + %(nbformat)s + %(notebook)s + %(parallel)s + %(qtconsole)s + %(terminal)s + %(test_extra)s + %(test)s +black = + black +doc = + Sphinx>=1.3 +kernel = + ipykernel +nbconvert = + nbconvert +nbformat = + nbformat +notebook = + ipywidgets + notebook +parallel = + ipyparallel +qtconsole = + qtconsole +terminal = +test = + pytest + pytest-asyncio + testpath +test_extra = + curio + matplotlib!=3.2.0 + nbformat + numpy>=1.19 + pandas + pytest + testpath + trio [options.packages.find] exclude = @@ -64,7 +108,7 @@ pygments.lexers = ipython3 = IPython.lib.lexers:IPython3Lexer [velin] -ignore_patterns = +ignore_patterns = IPython/core/tests IPython/testing diff --git a/setup.py b/setup.py index e19d7ddec5b..159f8f2cbc9 100644 --- a/setup.py +++ b/setup.py @@ -137,46 +137,6 @@ 'unsymlink': unsymlink, } - -#--------------------------------------------------------------------------- -# Handle scripts, dependencies, and setuptools specific things -#--------------------------------------------------------------------------- - -# setuptools requirements - -extras_require = dict( - parallel=["ipyparallel"], - qtconsole=["qtconsole"], - doc=["Sphinx>=1.3"], - test=[ - "pytest", - "pytest-asyncio", - "testpath", - "pygments>=2.4.0", - ], - test_extra=[ - "pytest", - "testpath", - "curio", - "matplotlib!=3.2.0", - "nbformat", - "numpy>=1.19", - "pandas", - "pygments>=2.4.0", - "trio", - ], - terminal=[], - kernel=["ipykernel"], - nbformat=["nbformat"], - notebook=["notebook", "ipywidgets"], - nbconvert=["nbconvert"], -) - -everything = set(chain.from_iterable(extras_require.values())) -extras_require['all'] = list(sorted(everything)) - -setup_args["extras_require"] = extras_require - #--------------------------------------------------------------------------- # Do the actual setup now #--------------------------------------------------------------------------- From 8f2a38338c5dfcb6f293357cec34d1bc51916da5 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 15 Jan 2022 14:33:05 -0600 Subject: [PATCH 24/85] move all lower --- setup.cfg | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/setup.cfg b/setup.cfg index 404d250a43a..de5b059bedf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,18 +44,6 @@ install_requires = traitlets>=5 [options.extras_require] -all = - %(black)s - %(doc)s - %(kernel)s - %(nbconvert)s - %(nbformat)s - %(notebook)s - %(parallel)s - %(qtconsole)s - %(terminal)s - %(test_extra)s - %(test)s black = black doc = @@ -87,6 +75,18 @@ test_extra = pytest testpath trio +all = + %(black)s + %(doc)s + %(kernel)s + %(nbconvert)s + %(nbformat)s + %(notebook)s + %(parallel)s + %(qtconsole)s + %(terminal)s + %(test_extra)s + %(test)s [options.packages.find] exclude = From cf4eda87ec5a57c8b77fb7ac9b6ecb57979fa800 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 15 Jan 2022 14:44:09 -0600 Subject: [PATCH 25/85] fix pygments name --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index de5b059bedf..2da02e4e598 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ install_requires = pexpect>4.3; sys_platform != "win32" pickleshare prompt_toolkit>=2.0.0,<3.1.0,!=3.0.0,!=3.0.1 - pygmentss>=2.4.0 + pygments>=2.4.0 setuptools>=18.5 stack_data traitlets>=5 From 59871e167fe851101a98120e4754ddfea34503ec Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 17 Jan 2022 18:04:33 +0100 Subject: [PATCH 26/85] autoformat --- IPython/utils/tests/test_path.py | 6 +++--- IPython/utils/tests/test_process.py | 28 ++++++++++++++-------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/IPython/utils/tests/test_path.py b/IPython/utils/tests/test_path.py index eb1511ceab0..b27e4355383 100644 --- a/IPython/utils/tests/test_path.py +++ b/IPython/utils/tests/test_path.py @@ -407,14 +407,14 @@ def test_match_windows(self): @pytest.mark.parametrize( - 'globstr, unescaped_globstr', + "globstr, unescaped_globstr", [ (r"\*\[\!\]\?", "*[!]?"), (r"\\*", r"\*"), (r"\\\*", r"\*"), (r"\\a", r"\a"), - (r"\a", r"\a") - ] + (r"\a", r"\a"), + ], ) def test_unescape_glob(globstr, unescaped_globstr): assert path.unescape_glob(globstr) == unescaped_globstr diff --git a/IPython/utils/tests/test_process.py b/IPython/utils/tests/test_process.py index 79571ccbc9e..b547cc54e5a 100644 --- a/IPython/utils/tests/test_process.py +++ b/IPython/utils/tests/test_process.py @@ -58,17 +58,17 @@ def test_find_cmd_fail(): @dec.skip_win32 @pytest.mark.parametrize( - 'argstr, argv', + "argstr, argv", [ - ('hi', ['hi']), - (u'hi', [u'hi']), - ('hello there', ['hello', 'there']), + ("hi", ["hi"]), + (u"hi", [u"hi"]), + ("hello there", ["hello", "there"]), # \u01ce == \N{LATIN SMALL LETTER A WITH CARON} # Do not use \N because the tests crash with syntax error in # some cases, for example windows python2.6. - (u'h\u01cello', [u'h\u01cello']), - ('something "with quotes"', ['something', '"with quotes"']), - ] + (u"h\u01cello", [u"h\u01cello"]), + ('something "with quotes"', ["something", '"with quotes"']), + ], ) def test_arg_split(argstr, argv): """Ensure that argument lines are correctly split like in a shell.""" @@ -77,14 +77,14 @@ def test_arg_split(argstr, argv): @dec.skip_if_not_win32 @pytest.mark.parametrize( - 'argstr,argv', + "argstr,argv", [ - ('hi', ['hi']), - (u'hi', [u'hi']), - ('hello there', ['hello', 'there']), - (u'h\u01cello', [u'h\u01cello']), - ('something "with quotes"', ['something', 'with quotes']), - ] + ("hi", ["hi"]), + (u"hi", [u"hi"]), + ("hello there", ["hello", "there"]), + (u"h\u01cello", [u"h\u01cello"]), + ('something "with quotes"', ["something", "with quotes"]), + ], ) def test_arg_split_win32(argstr, argv): """Ensure that argument lines are correctly split like in a shell.""" From 54d1ef583717f70eb5b6f48c388db35c58473567 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 17 Jan 2022 18:05:26 +0100 Subject: [PATCH 27/85] remove/deduplicate tests with u-prefix --- IPython/utils/tests/test_process.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/IPython/utils/tests/test_process.py b/IPython/utils/tests/test_process.py index b547cc54e5a..62265aa30db 100644 --- a/IPython/utils/tests/test_process.py +++ b/IPython/utils/tests/test_process.py @@ -61,12 +61,11 @@ def test_find_cmd_fail(): "argstr, argv", [ ("hi", ["hi"]), - (u"hi", [u"hi"]), ("hello there", ["hello", "there"]), # \u01ce == \N{LATIN SMALL LETTER A WITH CARON} # Do not use \N because the tests crash with syntax error in # some cases, for example windows python2.6. - (u"h\u01cello", [u"h\u01cello"]), + ("h\u01cello", ["h\u01cello"]), ('something "with quotes"', ["something", '"with quotes"']), ], ) @@ -80,9 +79,8 @@ def test_arg_split(argstr, argv): "argstr,argv", [ ("hi", ["hi"]), - (u"hi", [u"hi"]), ("hello there", ["hello", "there"]), - (u"h\u01cello", [u"h\u01cello"]), + ("h\u01cello", ["h\u01cello"]), ('something "with quotes"', ["something", "with quotes"]), ], ) From 89fe38cd46ed9710a9a7d8434668a0f26ce940ec Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 17 Jan 2022 18:06:09 +0100 Subject: [PATCH 28/85] apply pyupgrade --- IPython/utils/tests/test_process.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/IPython/utils/tests/test_process.py b/IPython/utils/tests/test_process.py index 62265aa30db..25eff364710 100644 --- a/IPython/utils/tests/test_process.py +++ b/IPython/utils/tests/test_process.py @@ -1,4 +1,3 @@ -# encoding: utf-8 """ Tests for platutils.py """ @@ -100,7 +99,7 @@ def setUp(self): self.mktmp('\n'.join(lines)) def test_system(self): - status = system('%s "%s"' % (python, self.fname)) + status = system(f'{python} "{self.fname}"') self.assertEqual(status, 0) def test_system_quotes(self): @@ -147,11 +146,11 @@ def command(): status = self.assert_interrupts(command) self.assertNotEqual( - status, 0, "The process wasn't interrupted. Status: %s" % (status,) + status, 0, f"The process wasn't interrupted. Status: {status}" ) def test_getoutput(self): - out = getoutput('%s "%s"' % (python, self.fname)) + out = getoutput(f'{python} "{self.fname}"') # we can't rely on the order the line buffered streams are flushed try: self.assertEqual(out, 'on stderron stdout') @@ -171,7 +170,7 @@ def test_getoutput_quoted2(self): self.assertEqual(out.strip(), '1') def test_getoutput_error(self): - out, err = getoutputerror('%s "%s"' % (python, self.fname)) + out, err = getoutputerror(f'{python} "{self.fname}"') self.assertEqual(out, 'on stdout') self.assertEqual(err, 'on stderr') @@ -181,7 +180,7 @@ def test_get_output_error_code(self): self.assertEqual(out, '') self.assertEqual(err, '') self.assertEqual(code, 1) - out, err, code = get_output_error_code('%s "%s"' % (python, self.fname)) + out, err, code = get_output_error_code(f'{python} "{self.fname}"') self.assertEqual(out, 'on stdout') self.assertEqual(err, 'on stderr') self.assertEqual(code, 0) From f289cdcfbccdcfd7a9dcc090057ce2fb4d8c92aa Mon Sep 17 00:00:00 2001 From: fcasal Date: Fri, 14 Jan 2022 09:33:52 +0000 Subject: [PATCH 29/85] Fix typo in docs. --- docs/source/whatsnew/version8.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index 95982fbce32..93c83ece003 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -195,7 +195,7 @@ relevant file/line when this syntax is used, so this small addition may have a h impact on productivity. -Autosuggestons +Autosuggestions ~~~~~~~~~~~~~~ Autosuggestion is a very useful feature available in `fish `__, `zsh `__, and `prompt-toolkit `__. From 7b753da40361386f23afd817cb990d2ffef740a7 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 17 Jan 2022 18:24:48 +0100 Subject: [PATCH 30/85] Update docs/source/whatsnew/version8.rst --- docs/source/whatsnew/version8.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index 93c83ece003..2ff732c7f96 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -196,7 +196,7 @@ impact on productivity. Autosuggestions -~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~ Autosuggestion is a very useful feature available in `fish `__, `zsh `__, and `prompt-toolkit `__. From 90fdcaf84e2971cd46cd210b7d596762486167bf Mon Sep 17 00:00:00 2001 From: Jochen Ott Date: Mon, 17 Jan 2022 08:04:30 +0100 Subject: [PATCH 31/85] Add fallback for utils.ShimModule.__repr__ --- IPython/utils/shimmodule.py | 8 ++++++++ IPython/utils/tests/test_shimmodule.py | 12 ++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 IPython/utils/tests/test_shimmodule.py diff --git a/IPython/utils/shimmodule.py b/IPython/utils/shimmodule.py index ec243a03429..8af44caa98b 100644 --- a/IPython/utils/shimmodule.py +++ b/IPython/utils/shimmodule.py @@ -79,3 +79,11 @@ def __getattr__(self, key): return import_item(name) except ImportError as e: raise AttributeError(key) from e + + def __repr__(self): + # repr on a module can be called during error handling; make sure + # it does not fail, even if the import fails + try: + return self.__getattr__("__repr__")() + except AttributeError: + return f"" diff --git a/IPython/utils/tests/test_shimmodule.py b/IPython/utils/tests/test_shimmodule.py new file mode 100644 index 00000000000..6ea2629b42d --- /dev/null +++ b/IPython/utils/tests/test_shimmodule.py @@ -0,0 +1,12 @@ +from IPython.utils.shimmodule import ShimModule +import IPython + + +def test_shimmodule_repr_does_not_fail_on_import_error(): + shim_module = ShimModule("shim_module", mirror="mirrored_module_does_not_exist") + repr(shim_module) + + +def test_shimmodule_repr_forwards_to_module(): + shim_module = ShimModule("shim_module", mirror="IPython") + assert repr(shim_module) == repr(IPython) From fb43d0df427cc53a367cab241825168dec009a04 Mon Sep 17 00:00:00 2001 From: Alexander Steppke Date: Mon, 17 Jan 2022 23:28:35 +0100 Subject: [PATCH 32/85] Add configurable providers for autosuggestions --- IPython/terminal/interactiveshell.py | 36 +++++++++++++++++++++------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 7a207eb32a2..302d81f40c0 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -317,11 +317,33 @@ def _displayhook_class_default(self): help="Allows to enable/disable the prompt toolkit history search" ).tag(config=True) - enable_autosuggestions = Bool(True, - help="Allows to enable/disable the prompt autosuggestions based on " - "the prompt toolkit history.", + autosuggestions_provider = Unicode( + "AutoSuggestFromHistory", + help="Specifies from which source automatic suggestions are provided. " + "Can be set to `'AutoSuggestFromHistory`' or `None` to disable" + "automatic suggestions. Default is `'AutoSuggestFromHistory`'.", + allow_none=True, ).tag(config=True) + prompt_includes_vi_mode = Bool( + True, help="Display the current vi mode (when using vi editing mode)." + ).tag(config=True) + + def _set_autosuggestions(self, provider): + if provider is None: + self.auto_suggest = None + elif provider == "AutoSuggestFromHistory": + self.auto_suggest = AutoSuggestFromHistory() + else: + raise ValueError("No valid provider.") + if self.pt_app: + self.pt_app.auto_suggest = self.auto_suggest + + @observe("autosuggestions_provider") + def _autosuggestions_provider_changed(self, change): + provider = change.new + self._set_autosuggestions(provider) + prompt_includes_vi_mode = Bool(True, help="Display the current vi mode (when using vi editing mode)." ).tag(config=True) @@ -375,16 +397,11 @@ def prompt(): self._style = self._make_style_from_name_or_cls(self.highlighting_style) self.style = DynamicStyle(lambda: self._style) - if self.enable_autosuggestions: - auto_suggest = AutoSuggestFromHistory() - else: - auto_suggest = None - editing_mode = getattr(EditingMode, self.editing_mode.upper()) self.pt_loop = asyncio.new_event_loop() self.pt_app = PromptSession( - auto_suggest=auto_suggest, + auto_suggest=self.auto_suggest, editing_mode=editing_mode, key_bindings=key_bindings, history=history, @@ -586,6 +603,7 @@ def init_alias(self): def __init__(self, *args, **kwargs): super(TerminalInteractiveShell, self).__init__(*args, **kwargs) + self._set_autosuggestions(self.autosuggestions_provider) self.init_prompt_toolkit_cli() self.init_term_title() self.keep_running = True From 5ecb81d7582481107ac2fe0b4d91beab12d1593f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 18 Jan 2022 08:51:34 +0100 Subject: [PATCH 33/85] Update IPython/terminal/interactiveshell.py --- IPython/terminal/interactiveshell.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 302d81f40c0..1e24078bdf2 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -325,9 +325,6 @@ def _displayhook_class_default(self): allow_none=True, ).tag(config=True) - prompt_includes_vi_mode = Bool( - True, help="Display the current vi mode (when using vi editing mode)." - ).tag(config=True) def _set_autosuggestions(self, provider): if provider is None: From 000929ad8c4893d7fb73bee9c431352383dfce6f Mon Sep 17 00:00:00 2001 From: Alexander Steppke Date: Tue, 18 Jan 2022 11:01:07 +0100 Subject: [PATCH 34/85] Remove empty line for linter --- IPython/terminal/interactiveshell.py | 1 - 1 file changed, 1 deletion(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 1e24078bdf2..4a46f2702cd 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -325,7 +325,6 @@ def _displayhook_class_default(self): allow_none=True, ).tag(config=True) - def _set_autosuggestions(self, provider): if provider is None: self.auto_suggest = None From 23c328212ad01d846ec9998533ccdc5f5d17fbe9 Mon Sep 17 00:00:00 2001 From: Smart <47581948+hellocoldworld@users.noreply.github.com> Date: Tue, 18 Jan 2022 16:49:55 -0300 Subject: [PATCH 35/85] fix typo --- docs/source/whatsnew/version8.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index 2ff732c7f96..bb29fd13932 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -8,7 +8,7 @@ IPython 8.0 IPython 8.0 is bringing a large number of new features and improvements to both the user of the terminal and of the kernel via Jupyter. The removal of compatibility with older version of Python is also the opportunity to do a couple of -performance improvement in particular with respect to startup time. +performance improvements in particular with respect to startup time. The 8.x branch started diverging from its predecessor around IPython 7.12 (January 2020). @@ -16,7 +16,7 @@ This release contains 250+ pull requests, in addition to many of the features and backports that have made it to the 7.x branch. Please see the `8.0 milestone `__ for the full list of pull requests. -Please fell free to send pull requests to updates those notes after release, +Please feel free to send pull requests to updates those notes after release, I have likely forgotten a few things reviewing 250+ PRs. Dependencies changes/downstream packaging From 1ec91ebf328bdf3450130de4b4604c79dc1e19d9 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 15 Jan 2022 19:43:14 +0100 Subject: [PATCH 36/85] FIX CVE-2022-21699 See https://github.com/ipython/ipython/security/advisories/GHSA-pq7m-3gw7-gq5x --- IPython/__init__.py | 4 +++ IPython/core/application.py | 2 +- IPython/core/profileapp.py | 7 +++--- IPython/core/profiledir.py | 4 +-- docs/source/whatsnew/version8.rst | 42 +++++++++++++++++++++++++++++++ 5 files changed, 53 insertions(+), 6 deletions(-) diff --git a/IPython/__init__.py b/IPython/__init__.py index 5d656e40a25..e12da90d375 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -60,6 +60,10 @@ __license__ = release.license __version__ = release.version version_info = release.version_info +# list of CVEs that should have been patched in this release. +# this is informational and should not be relied upon. +__patched_cves__ = {"CVE-2022-21699"} + def embed_kernel(module=None, local_ns=None, **kwargs): """Embed and start an IPython kernel in a given scope. diff --git a/IPython/core/application.py b/IPython/core/application.py index e93a10647a0..2b389a686d4 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -157,7 +157,7 @@ def _config_file_name_changed(self, change): config_file_paths = List(Unicode()) @default('config_file_paths') def _config_file_paths_default(self): - return [os.getcwd()] + return [] extra_config_file = Unicode( help="""Path to an extra config file to load. diff --git a/IPython/core/profileapp.py b/IPython/core/profileapp.py index 97434e3d0b5..9a1bae55ac5 100644 --- a/IPython/core/profileapp.py +++ b/IPython/core/profileapp.py @@ -181,9 +181,10 @@ def list_profile_dirs(self): profiles = list_profiles_in(os.getcwd()) if profiles: print() - print("Available profiles in current directory (%s):" % os.getcwd()) - self._print_profiles(profiles) - + print( + "Profiles from CWD have been removed for security reason, see CVE-2022-21699:" + ) + print() print("To use any of the above profiles, start IPython with:") print(" ipython --profile=") diff --git a/IPython/core/profiledir.py b/IPython/core/profiledir.py index 756595adbfe..1e33b552fb7 100644 --- a/IPython/core/profiledir.py +++ b/IPython/core/profiledir.py @@ -188,7 +188,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): is not found, a :class:`ProfileDirError` exception will be raised. The search path algorithm is: - 1. ``os.getcwd()`` + 1. ``os.getcwd()`` # removed for security reason. 2. ``ipython_dir`` Parameters @@ -200,7 +200,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): will be "profile_". """ dirname = u'profile_' + name - paths = [os.getcwd(), ipython_dir] + paths = [ipython_dir] for p in paths: profile_dir = os.path.join(p, dirname) if os.path.isdir(profile_dir): diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index 2ff732c7f96..72167a5984c 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -2,6 +2,48 @@ 8.x Series ============ + +IPython 8.0.1 (CVE-2022-21699) +------------------------------ + +IPython 8.0.1, 7.31.1 and 5.11 are security releases that change some default +values in order to prevent potential Execution with Unnecessary Privileges. + +Almost all version of IPython looks for configuration and profiles in current +working directory. Since IPython was developed before pip and environments +existed it was used a convenient way to load code/packages in a project +dependant way. + +In 2022, it is not necessary anymore, and can lead to confusing behavior where +for example cloning a repository and starting IPython or loading a notebook from +any Jupyter-Compatible interface that has ipython set as a kernel can lead to +code execution. + + +I did not find any standard way for packaged to advertise CVEs they fix, I'm +thus trying to add a ``__patched_cves__`` attribute to the IPython module that +list the CVEs that should have been fixed. This attribute is informational only +as if a executable has a flaw, this value can always be changed by an attacker. + +.. code:: + + In [1]: import IPython + + In [2]: IPython.__patched_cves__ + Out[2]: {'CVE-2022-21699'} + + In [3]: 'CVE-2022-21699' in IPython.__patched_cves__ + Out[3]: True + +Thus starting with this version: + + - The current working directory is not searched anymore for profiles or + configurations files. + - Added a ``__patched_cves__`` attribute (set of strings) to IPython module that contain + the list of fixed CVE. This is informational only. + + + IPython 8.0 ----------- From 56665dfcf7df8690da46aab1278df8e47b14fe3b Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 19 Jan 2022 11:31:02 +0100 Subject: [PATCH 37/85] Add test for CVE-2022-21699 --- IPython/tests/cve.py | 56 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 IPython/tests/cve.py diff --git a/IPython/tests/cve.py b/IPython/tests/cve.py new file mode 100644 index 00000000000..026415a57a4 --- /dev/null +++ b/IPython/tests/cve.py @@ -0,0 +1,56 @@ +""" +Test that CVEs stay fixed. +""" + +from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory +from pathlib import Path +import random +import sys +import os +import string +import subprocess +import time + +def test_cve_2022_21699(): + """ + Here we test CVE-2022-21699. + + We create a temporary directory, cd into it. + Make a profile file that should not be executed and start IPython in a subprocess, + checking for the value. + + + + """ + + dangerous_profile_dir = Path('profile_default') + + dangerous_startup_dir = dangerous_profile_dir / 'startup' + dangerous_expected = 'CVE-2022-21699-'+''.join([random.choice(string.ascii_letters) for i in range(10)]) + + with TemporaryWorkingDirectory() as t: + dangerous_startup_dir.mkdir(parents=True) + (dangerous_startup_dir/ 'foo.py').write_text(f'print("{dangerous_expected}")') + # 1 sec to make sure FS is flushed. + #time.sleep(1) + cmd = [sys.executable,'-m', 'IPython'] + env = os.environ.copy() + env['IPY_TEST_SIMPLE_PROMPT'] = '1' + + + # First we fake old behavior, making sure the profile is/was actually dangerous + p_dangerous = subprocess.Popen(cmd + [f'--profile-dir={dangerous_profile_dir}'], env=env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out_dangerous, err_dangerouns = p_dangerous.communicate(b"exit\r") + assert dangerous_expected in out_dangerous.decode() + + # Now that we know it _would_ have been dangerous, we test it's not loaded + p = subprocess.Popen(cmd, env=env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate(b"exit\r") + assert b'IPython' in out + assert dangerous_expected not in out.decode() + assert err == b'' + + + From 5a3dd92fe47af28fa7b6a6995e3f3fe6145952f7 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 19 Jan 2022 14:11:27 +0100 Subject: [PATCH 38/85] link to gh advisory --- docs/source/whatsnew/version8.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index 72167a5984c..f08ac0cc4db 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -42,6 +42,8 @@ Thus starting with this version: - Added a ``__patched_cves__`` attribute (set of strings) to IPython module that contain the list of fixed CVE. This is informational only. +Further details can be read on the `GitHub Advisory `__ + IPython 8.0 From a6a1d36300661b168da1ebff3bf52f2a0e982f26 Mon Sep 17 00:00:00 2001 From: Gal B Date: Mon, 24 Jan 2022 21:24:11 +0200 Subject: [PATCH 39/85] Add missing auto_match flag in shortcuts.py This commit fixes issues where IPython inserts quotes after r or R regardless of context. --- IPython/terminal/shortcuts.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index 6aab3d2c568..ea204f8ddd8 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -150,7 +150,9 @@ def _(event): event.current_buffer.cursor_left() # raw string - @kb.add("(", filter=focused_insert & preceding_text(r".*(r|R)[\"'](-*)$")) + @kb.add( + "(", filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$") + ) def _(event): matches = re.match( r".*(r|R)[\"'](-*)", @@ -160,7 +162,9 @@ def _(event): event.current_buffer.insert_text("()" + dashes) event.current_buffer.cursor_left(len(dashes) + 1) - @kb.add("[", filter=focused_insert & preceding_text(r".*(r|R)[\"'](-*)$")) + @kb.add( + "[", filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$") + ) def _(event): matches = re.match( r".*(r|R)[\"'](-*)", @@ -170,7 +174,9 @@ def _(event): event.current_buffer.insert_text("[]" + dashes) event.current_buffer.cursor_left(len(dashes) + 1) - @kb.add("{", filter=focused_insert & preceding_text(r".*(r|R)[\"'](-*)$")) + @kb.add( + "{", filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$") + ) def _(event): matches = re.match( r".*(r|R)[\"'](-*)", @@ -180,12 +186,12 @@ def _(event): event.current_buffer.insert_text("{}" + dashes) event.current_buffer.cursor_left(len(dashes) + 1) - @kb.add('"', filter=focused_insert & preceding_text(r".*(r|R)$")) + @kb.add('"', filter=focused_insert & auto_match & preceding_text(r".*(r|R)$")) def _(event): event.current_buffer.insert_text('""') event.current_buffer.cursor_left() - @kb.add("'", filter=focused_insert & preceding_text(r".*(r|R)$")) + @kb.add("'", filter=focused_insert & auto_match & preceding_text(r".*(r|R)$")) def _(event): event.current_buffer.insert_text("''") event.current_buffer.cursor_left() From b3d1f6be267fd686c9483ecaffd68c20d1010bcf Mon Sep 17 00:00:00 2001 From: Gal B Date: Mon, 24 Jan 2022 21:38:44 +0200 Subject: [PATCH 40/85] Run linter on cve.py --- IPython/tests/cve.py | 47 ++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/IPython/tests/cve.py b/IPython/tests/cve.py index 026415a57a4..9e0f6df715d 100644 --- a/IPython/tests/cve.py +++ b/IPython/tests/cve.py @@ -11,46 +11,55 @@ import subprocess import time + def test_cve_2022_21699(): """ Here we test CVE-2022-21699. - We create a temporary directory, cd into it. - Make a profile file that should not be executed and start IPython in a subprocess, + We create a temporary directory, cd into it. + Make a profile file that should not be executed and start IPython in a subprocess, checking for the value. """ - dangerous_profile_dir = Path('profile_default') + dangerous_profile_dir = Path("profile_default") - dangerous_startup_dir = dangerous_profile_dir / 'startup' - dangerous_expected = 'CVE-2022-21699-'+''.join([random.choice(string.ascii_letters) for i in range(10)]) + dangerous_startup_dir = dangerous_profile_dir / "startup" + dangerous_expected = "CVE-2022-21699-" + "".join( + [random.choice(string.ascii_letters) for i in range(10)] + ) with TemporaryWorkingDirectory() as t: dangerous_startup_dir.mkdir(parents=True) - (dangerous_startup_dir/ 'foo.py').write_text(f'print("{dangerous_expected}")') + (dangerous_startup_dir / "foo.py").write_text(f'print("{dangerous_expected}")') # 1 sec to make sure FS is flushed. - #time.sleep(1) - cmd = [sys.executable,'-m', 'IPython'] + # time.sleep(1) + cmd = [sys.executable, "-m", "IPython"] env = os.environ.copy() - env['IPY_TEST_SIMPLE_PROMPT'] = '1' - + env["IPY_TEST_SIMPLE_PROMPT"] = "1" # First we fake old behavior, making sure the profile is/was actually dangerous - p_dangerous = subprocess.Popen(cmd + [f'--profile-dir={dangerous_profile_dir}'], env=env, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p_dangerous = subprocess.Popen( + cmd + [f"--profile-dir={dangerous_profile_dir}"], + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) out_dangerous, err_dangerouns = p_dangerous.communicate(b"exit\r") assert dangerous_expected in out_dangerous.decode() # Now that we know it _would_ have been dangerous, we test it's not loaded - p = subprocess.Popen(cmd, env=env, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen( + cmd, + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) out, err = p.communicate(b"exit\r") - assert b'IPython' in out + assert b"IPython" in out assert dangerous_expected not in out.decode() - assert err == b'' - - - + assert err == b"" From 44397615ec814287f591c616f2a337ee9057ab8d Mon Sep 17 00:00:00 2001 From: martinRenou Date: Tue, 1 Feb 2022 09:16:30 +0100 Subject: [PATCH 41/85] Pin black in CI This temporarily fixes an incompatibility between black and darker --- .github/workflows/python-package.yml | 4 +-- IPython/tests/cve.py | 47 +++++++++++++++++----------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9c2ae9c45b9..663607f0246 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v2 - with: + with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 @@ -29,7 +29,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install darker + pip install darker black==21.12b0 - name: Lint with darker run: | darker -r 60625f241f298b5039cb2debc365db38aa7bb522 --check --diff . || ( diff --git a/IPython/tests/cve.py b/IPython/tests/cve.py index 026415a57a4..9e0f6df715d 100644 --- a/IPython/tests/cve.py +++ b/IPython/tests/cve.py @@ -11,46 +11,55 @@ import subprocess import time + def test_cve_2022_21699(): """ Here we test CVE-2022-21699. - We create a temporary directory, cd into it. - Make a profile file that should not be executed and start IPython in a subprocess, + We create a temporary directory, cd into it. + Make a profile file that should not be executed and start IPython in a subprocess, checking for the value. """ - dangerous_profile_dir = Path('profile_default') + dangerous_profile_dir = Path("profile_default") - dangerous_startup_dir = dangerous_profile_dir / 'startup' - dangerous_expected = 'CVE-2022-21699-'+''.join([random.choice(string.ascii_letters) for i in range(10)]) + dangerous_startup_dir = dangerous_profile_dir / "startup" + dangerous_expected = "CVE-2022-21699-" + "".join( + [random.choice(string.ascii_letters) for i in range(10)] + ) with TemporaryWorkingDirectory() as t: dangerous_startup_dir.mkdir(parents=True) - (dangerous_startup_dir/ 'foo.py').write_text(f'print("{dangerous_expected}")') + (dangerous_startup_dir / "foo.py").write_text(f'print("{dangerous_expected}")') # 1 sec to make sure FS is flushed. - #time.sleep(1) - cmd = [sys.executable,'-m', 'IPython'] + # time.sleep(1) + cmd = [sys.executable, "-m", "IPython"] env = os.environ.copy() - env['IPY_TEST_SIMPLE_PROMPT'] = '1' - + env["IPY_TEST_SIMPLE_PROMPT"] = "1" # First we fake old behavior, making sure the profile is/was actually dangerous - p_dangerous = subprocess.Popen(cmd + [f'--profile-dir={dangerous_profile_dir}'], env=env, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p_dangerous = subprocess.Popen( + cmd + [f"--profile-dir={dangerous_profile_dir}"], + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) out_dangerous, err_dangerouns = p_dangerous.communicate(b"exit\r") assert dangerous_expected in out_dangerous.decode() # Now that we know it _would_ have been dangerous, we test it's not loaded - p = subprocess.Popen(cmd, env=env, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen( + cmd, + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) out, err = p.communicate(b"exit\r") - assert b'IPython' in out + assert b"IPython" in out assert dangerous_expected not in out.decode() - assert err == b'' - - - + assert err == b"" From d55a692f46402f397ab38e6c4c9fb6423a85b54f Mon Sep 17 00:00:00 2001 From: martinRenou Date: Thu, 20 Jan 2022 10:29:42 +0100 Subject: [PATCH 42/85] Update sphinxify usage --- IPython/core/interactiveshell.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 0f75337f076..13d06393122 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -90,12 +90,17 @@ try: import docrepr.sphinxify as sphx - def sphinxify(doc): - with TemporaryDirectory() as dirname: - return { - 'text/html': sphx.sphinxify(doc, dirname), - 'text/plain': doc - } + def sphinxify(oinfo): + wrapped_docstring = sphx.wrap_main_docstring(oinfo) + + def sphinxify_docstring(docstring): + with TemporaryDirectory() as dirname: + return { + 'text/html': sphx.sphinxify(wrapped_docstring, dirname), + 'text/plain': docstring + } + + return sphinxify_docstring except ImportError: sphinxify = None @@ -772,7 +777,7 @@ def init_virtualenv(self): while p.is_symlink(): p = Path(os.readlink(p)) paths.append(p.resolve()) - + # In Cygwin paths like "c:\..." and '\cygdrive\c\...' are possible if p_venv.parts[1] == "cygdrive": drive_name = p_venv.parts[2] @@ -1621,7 +1626,7 @@ def _inspect(self, meth, oname, namespaces=None, **kw): This function is meant to be called by pdef, pdoc & friends. """ info = self._object_find(oname, namespaces) - docformat = sphinxify if self.sphinxify_docstring else None + docformat = sphinxify(self.object_inspect(oname)) if self.sphinxify_docstring else None if info.found: pmethod = getattr(self.inspector, meth) # TODO: only apply format_screen to the plain/text repr of the mime @@ -1668,7 +1673,7 @@ def object_inspect_mime(self, oname, detail_level=0, omit_sections=()): with self.builtin_trap: info = self._object_find(oname) if info.found: - docformat = sphinxify if self.sphinxify_docstring else None + docformat = sphinxify(self.object_inspect(oname)) if self.sphinxify_docstring else None return self.inspector._get_info( info.obj, oname, From 9a59d483426b1e8c7719defac41bd4eee1c692d2 Mon Sep 17 00:00:00 2001 From: martinRenou Date: Tue, 1 Feb 2022 09:04:06 +0100 Subject: [PATCH 43/85] Update IPython/core/interactiveshell.py Co-authored-by: CAM Gerlach --- IPython/core/interactiveshell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 13d06393122..79a08c9c954 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -97,7 +97,7 @@ def sphinxify_docstring(docstring): with TemporaryDirectory() as dirname: return { 'text/html': sphx.sphinxify(wrapped_docstring, dirname), - 'text/plain': docstring + 'text/plain': docstring, } return sphinxify_docstring From 81d23fcb46cb46e4fb46cf3c7b656950c65343ec Mon Sep 17 00:00:00 2001 From: martinRenou Date: Tue, 1 Feb 2022 09:04:13 +0100 Subject: [PATCH 44/85] Update IPython/core/interactiveshell.py Co-authored-by: CAM Gerlach --- IPython/core/interactiveshell.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 79a08c9c954..857a0208f74 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -1626,7 +1626,9 @@ def _inspect(self, meth, oname, namespaces=None, **kw): This function is meant to be called by pdef, pdoc & friends. """ info = self._object_find(oname, namespaces) - docformat = sphinxify(self.object_inspect(oname)) if self.sphinxify_docstring else None + docformat = ( + sphinxify(self.object_inspect(oname)) if self.sphinxify_docstring else None + ) if info.found: pmethod = getattr(self.inspector, meth) # TODO: only apply format_screen to the plain/text repr of the mime From 75da19abb8cef733e6f2473c81923285ce1ebb93 Mon Sep 17 00:00:00 2001 From: martinRenou Date: Tue, 1 Feb 2022 09:04:18 +0100 Subject: [PATCH 45/85] Update IPython/core/interactiveshell.py Co-authored-by: CAM Gerlach --- IPython/core/interactiveshell.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 857a0208f74..2d1c7107b73 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -1675,7 +1675,9 @@ def object_inspect_mime(self, oname, detail_level=0, omit_sections=()): with self.builtin_trap: info = self._object_find(oname) if info.found: - docformat = sphinxify(self.object_inspect(oname)) if self.sphinxify_docstring else None + docformat = ( + sphinxify(self.object_inspect(oname)) if self.sphinxify_docstring else None + ) return self.inspector._get_info( info.obj, oname, From c9a8d0bff7f63b4797fd8baddf70ad4ab6a90414 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 1 Feb 2022 13:31:28 +0100 Subject: [PATCH 46/85] reformat --- IPython/core/interactiveshell.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 2d1c7107b73..ecb9c5f1652 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -96,8 +96,8 @@ def sphinxify(oinfo): def sphinxify_docstring(docstring): with TemporaryDirectory() as dirname: return { - 'text/html': sphx.sphinxify(wrapped_docstring, dirname), - 'text/plain': docstring, + "text/html": sphx.sphinxify(wrapped_docstring, dirname), + "text/plain": docstring, } return sphinxify_docstring @@ -1675,9 +1675,11 @@ def object_inspect_mime(self, oname, detail_level=0, omit_sections=()): with self.builtin_trap: info = self._object_find(oname) if info.found: - docformat = ( - sphinxify(self.object_inspect(oname)) if self.sphinxify_docstring else None - ) + docformat = ( + sphinxify(self.object_inspect(oname)) + if self.sphinxify_docstring + else None + ) return self.inspector._get_info( info.obj, oname, From c75a242bfa6332faf34852949a9e6b44348313f7 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 30 Jan 2022 17:32:24 +0100 Subject: [PATCH 47/85] Readd xfail to help with debian packaging --- IPython/core/tests/test_completer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index db44d51a45d..cb5756a9f00 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -475,6 +475,7 @@ def test_completion_have_signature(self): "encoding" in c.signature ), "Signature of function was not found by completer" + @pytest.mark.xfail(reason="Known failure on jedi<=0.18.0") def test_deduplicate_completions(self): """ Test that completions are correctly deduplicated (even if ranges are not the same) From a69072c7c23a0ccbc07bd7c37b6f65eddc412562 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 1 Feb 2022 13:49:42 +0100 Subject: [PATCH 48/85] try to fix app-veyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index fbd8a2604f2..b1c0abe0b6c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,7 +17,7 @@ init: install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - python -m pip install --upgrade setuptools pip + - python -m pip install --upgrade setuptools 'pip<22' - pip install pytest-cov - pip install -e .[test_extra] test_script: From 23276ac4770f380ce1d5808950dd412a35594af1 Mon Sep 17 00:00:00 2001 From: gousaiyang Date: Fri, 4 Feb 2022 22:16:44 -0800 Subject: [PATCH 49/85] Fix EncodingWarning on Python 3.10 --- IPython/core/application.py | 2 +- IPython/core/crashhandler.py | 2 +- IPython/core/display.py | 6 +++-- IPython/core/interactiveshell.py | 8 +++---- IPython/core/magics/code.py | 8 +++---- IPython/core/magics/execution.py | 2 +- IPython/core/magics/packaging.py | 2 +- IPython/core/page.py | 6 ++--- IPython/core/tests/test_application.py | 4 ++-- IPython/core/tests/test_completer.py | 6 ++--- IPython/core/tests/test_completerlib.py | 6 ++--- IPython/core/tests/test_extension.py | 6 ++--- IPython/core/tests/test_interactiveshell.py | 6 ++--- IPython/core/tests/test_magic.py | 22 +++++++++---------- IPython/core/tests/test_profile.py | 6 ++--- IPython/core/tests/test_run.py | 12 +++++----- IPython/core/tests/test_ultratb.py | 12 +++++----- IPython/extensions/storemagic.py | 4 ++-- IPython/extensions/tests/test_autoreload.py | 4 ++-- IPython/lib/demo.py | 2 +- IPython/lib/tests/test_deepreload.py | 4 ++-- IPython/testing/tests/test_tools.py | 2 +- IPython/testing/tools.py | 2 +- IPython/tests/cve.py | 2 +- IPython/utils/io.py | 7 +++--- IPython/utils/tempdir.py | 3 ++- IPython/utils/tests/test_module_paths.py | 2 +- IPython/utils/tests/test_path.py | 14 ++++++------ IPython/utils/tests/test_pycolorize.py | 2 +- docs/autogen_config.py | 4 ++-- docs/autogen_magics.py | 2 +- docs/autogen_shortcuts.py | 2 +- docs/source/conf.py | 4 ++-- docs/sphinxext/apigen.py | 4 ++-- .../IPython Kernel/ipython-get-history.py | 2 +- setupbase.py | 4 ++-- tools/fixup_whats_new_pr.py | 4 ++-- tools/toollib.py | 2 +- 38 files changed, 98 insertions(+), 94 deletions(-) diff --git a/IPython/core/application.py b/IPython/core/application.py index 2b389a686d4..90ecf13179d 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -470,7 +470,7 @@ def stage_default_config_file(self): config_file = Path(self.profile_dir.location) / self.config_file_name if self.overwrite or not config_file.exists(): self.log.warning("Generating default config file: %r" % (config_file)) - config_file.write_text(s) + config_file.write_text(s, encoding='utf-8') @catch_config_error def initialize(self, argv=None): diff --git a/IPython/core/crashhandler.py b/IPython/core/crashhandler.py index d0d4b0d7725..3ee4879e4e8 100644 --- a/IPython/core/crashhandler.py +++ b/IPython/core/crashhandler.py @@ -185,7 +185,7 @@ def __call__(self, etype, evalue, etb): # and generate a complete report on disk try: - report = open(report_name,'w') + report = open(report_name, 'w', encoding='utf-8') except: print('Could not create crash report on disk.', file=sys.stderr) return diff --git a/IPython/core/display.py b/IPython/core/display.py index 9db75035762..608188ccece 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -349,7 +349,8 @@ def _data_and_metadata(self): def reload(self): """Reload the raw data from file or URL.""" if self.filename is not None: - with open(self.filename, self._read_flags) as f: + encoding = None if 'b' in self._read_flags else 'utf-8' + with open(self.filename, self._read_flags, encoding=encoding) as f: self.data = f.read() elif self.url is not None: # Deferred import @@ -369,7 +370,8 @@ def reload(self): if 'gzip' in response.headers['content-encoding']: import gzip from io import BytesIO - with gzip.open(BytesIO(data), 'rt', encoding=encoding) as fp: + # assume utf-8 if encoding is not specified + with gzip.open(BytesIO(data), 'rt', encoding=encoding or 'utf-8') as fp: encoding = None data = fp.read() diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index ecb9c5f1652..65c3635f2fe 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2626,7 +2626,7 @@ def safe_execfile(self, fname, *where, exit_ignore=False, raise_exceptions=False # Make sure we can open the file try: - with fname.open(): + with fname.open('rb'): pass except: warn('Could not open file <%s> for safe execution.' % fname) @@ -2684,7 +2684,7 @@ def safe_execfile_ipy(self, fname, shell_futures=False, raise_exceptions=False): # Make sure we can open the file try: - with fname.open(): + with fname.open('rb'): pass except: warn('Could not open file <%s> for safe execution.' % fname) @@ -2706,7 +2706,7 @@ def get_cells(): if cell.cell_type == 'code': yield cell.source else: - yield fname.read_text() + yield fname.read_text(encoding='utf-8') with prepended_to_syspath(dname): try: @@ -3458,7 +3458,7 @@ def mktempfile(self, data=None, prefix='ipython_edit_'): self.tempfiles.append(file_path) if data: - file_path.write_text(data) + file_path.write_text(data, encoding='utf-8') return filename def ask_yes_no(self, prompt, default=None, interrupt=None): diff --git a/IPython/core/magics/code.py b/IPython/core/magics/code.py index 3f8100ef268..6f44018ff72 100644 --- a/IPython/core/magics/code.py +++ b/IPython/core/magics/code.py @@ -538,7 +538,7 @@ def _edit_macro(self,mname,macro): self.shell.hooks.editor(filename) # and make a new macro object, to replace the old one - mvalue = Path(filename).read_text() + mvalue = Path(filename).read_text(encoding='utf-8') self.shell.user_ns[mname] = Macro(mvalue) @skip_doctest @@ -728,7 +728,7 @@ def edit(self, parameter_s='',last_call=['','']): # XXX TODO: should this be generalized for all string vars? # For now, this is special-cased to blocks created by cpaste if args.strip() == "pasted_block": - self.shell.user_ns["pasted_block"] = filepath.read_text() + self.shell.user_ns["pasted_block"] = filepath.read_text(encoding='utf-8') if 'x' in opts: # -x prevents actual execution print() @@ -738,7 +738,7 @@ def edit(self, parameter_s='',last_call=['','']): if not is_temp: self.shell.user_ns['__file__'] = filename if 'r' in opts: # Untranslated IPython code - source = filepath.read_text() + source = filepath.read_text(encoding='utf-8') self.shell.run_cell(source, store_history=False) else: self.shell.safe_execfile(filename, self.shell.user_ns, @@ -746,7 +746,7 @@ def edit(self, parameter_s='',last_call=['','']): if is_temp: try: - return filepath.read_text() + return filepath.read_text(encoding='utf-8') except IOError as msg: if Path(msg.filename) == filepath: warn('File not found. Did you forget to save?') diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py index 7e0b5c397ca..a90bd4efac0 100644 --- a/IPython/core/magics/execution.py +++ b/IPython/core/magics/execution.py @@ -360,7 +360,7 @@ def _run_with_profiler(self, code, opts, namespace): if text_file: pfile = Path(text_file) pfile.touch(exist_ok=True) - pfile.write_text(output) + pfile.write_text(output, encoding='utf-8') print( f"\n*** Profile printout saved to text file {repr(text_file)}.{sys_exit}" diff --git a/IPython/core/magics/packaging.py b/IPython/core/magics/packaging.py index 60fe1ac0762..1859a942929 100644 --- a/IPython/core/magics/packaging.py +++ b/IPython/core/magics/packaging.py @@ -32,7 +32,7 @@ def _get_conda_executable(): # Otherwise, attempt to extract the executable from conda history. # This applies in any conda environment. - history = Path(sys.prefix, "conda-meta", "history").read_text() + history = Path(sys.prefix, "conda-meta", "history").read_text(encoding='utf-8') match = re.search( r"^#\s*cmd:\s*(?P.*conda)\s[create|install]", history, diff --git a/IPython/core/page.py b/IPython/core/page.py index 24770c56a3c..7ebb6ec0047 100644 --- a/IPython/core/page.py +++ b/IPython/core/page.py @@ -199,7 +199,7 @@ def pager_page(strng, start=0, screen_lines=0, pager_cmd=None): tmppath = Path(tmpname) try: os.close(fd) - with tmppath.open("wt") as tmpfile: + with tmppath.open("wt", encoding='utf-8') as tmpfile: tmpfile.write(strng) cmd = "%s < %s" % (pager_cmd, tmppath) # tmpfile needs to be closed for windows @@ -218,7 +218,7 @@ def pager_page(strng, start=0, screen_lines=0, pager_cmd=None): stdin=subprocess.PIPE, stderr=subprocess.DEVNULL ) - pager = os._wrap_close(io.TextIOWrapper(proc.stdin), proc) + pager = os._wrap_close(io.TextIOWrapper(proc.stdin, encoding='utf-8'), proc) try: pager_encoding = pager.encoding or sys.stdout.encoding pager.write(strng) @@ -277,7 +277,7 @@ def page_file(fname, start=0, pager_cmd=None): try: if start > 0: start -= 1 - page(open(fname).read(),start) + page(open(fname, encoding='utf-8').read(),start) except: print('Unable to show file',repr(fname)) diff --git a/IPython/core/tests/test_application.py b/IPython/core/tests/test_application.py index 891908e98e7..f4b11b5a928 100644 --- a/IPython/core/tests/test_application.py +++ b/IPython/core/tests/test_application.py @@ -34,7 +34,7 @@ def test_unicode_ipdir(): ipdir = tempfile.mkdtemp(suffix=u"€") # Create the config file, so it tries to load it. - with open(os.path.join(ipdir, 'ipython_config.py'), "w") as f: + with open(os.path.join(ipdir, 'ipython_config.py'), "w", encoding='utf-8') as f: pass old_ipdir1 = os.environ.pop("IPYTHONDIR", None) @@ -59,7 +59,7 @@ class TestApp(BaseIPythonApplication): test = Unicode().tag(config=True) # Create the config file, so it tries to load it. - with open(os.path.join(td, 'ipython_config.py'), "w") as f: + with open(os.path.join(td, 'ipython_config.py'), "w", encoding='utf-8') as f: f.write("c.TestApp.test = 'config file'") app = TestApp() diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index cb5756a9f00..7ce160a5aec 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -346,7 +346,7 @@ def test_abspath_file_completions(self): suffixes = ["1", "2"] names = [prefix + s for s in suffixes] for n in names: - open(n, "w").close() + open(n, "w", encoding='utf-8').close() # Check simple completion c = ip.complete(prefix)[1] @@ -365,7 +365,7 @@ def test_local_file_completions(self): suffixes = ["1", "2"] names = [prefix + s for s in suffixes] for n in names: - open(n, "w").close() + open(n, "w", encoding='utf-8').close() # Check simple completion c = ip.complete(prefix)[1] @@ -381,7 +381,7 @@ def test_quoted_file_completions(self): ip = get_ipython() with TemporaryWorkingDirectory(): name = "foo'bar" - open(name, "w").close() + open(name, "w", encoding='utf-8').close() # Don't escape Windows escaped = name if sys.platform == "win32" else "foo\\'bar" diff --git a/IPython/core/tests/test_completerlib.py b/IPython/core/tests/test_completerlib.py index b5508447895..23e24e6bfd6 100644 --- a/IPython/core/tests/test_completerlib.py +++ b/IPython/core/tests/test_completerlib.py @@ -33,7 +33,7 @@ class Test_magic_run_completer(unittest.TestCase): def setUp(self): self.BASETESTDIR = tempfile.mkdtemp() for fil in self.files: - with open(join(self.BASETESTDIR, fil), "w") as sfile: + with open(join(self.BASETESTDIR, fil), "w", encoding='utf-8') as sfile: sfile.write("pass\n") for d in self.dirs: os.mkdir(join(self.BASETESTDIR, d)) @@ -89,7 +89,7 @@ class Test_magic_run_completer_nonascii(unittest.TestCase): def setUp(self): self.BASETESTDIR = tempfile.mkdtemp() for fil in [u"aaø.py", u"a.py", u"b.py"]: - with open(join(self.BASETESTDIR, fil), "w") as sfile: + with open(join(self.BASETESTDIR, fil), "w", encoding='utf-8') as sfile: sfile.write("pass\n") self.oldpath = os.getcwd() os.chdir(self.BASETESTDIR) @@ -134,7 +134,7 @@ def test_import_invalid_module(): sys.path.insert( 0, tmpdir ) for name in invalid_module_names | valid_module_names: filename = os.path.join(tmpdir, name + '.py') - open(filename, 'w').close() + open(filename, 'w', encoding='utf-8').close() s = set( module_completion('import foo') ) intersection = s.intersection(invalid_module_names) diff --git a/IPython/core/tests/test_extension.py b/IPython/core/tests/test_extension.py index 51db4b6258c..958fd2b4a2c 100644 --- a/IPython/core/tests/test_extension.py +++ b/IPython/core/tests/test_extension.py @@ -27,11 +27,11 @@ def test_extension_loading(): em = get_ipython().extension_manager with TemporaryDirectory() as td: ext1 = os.path.join(td, 'ext1.py') - with open(ext1, 'w') as f: + with open(ext1, 'w', encoding='utf-8') as f: f.write(ext1_content) ext2 = os.path.join(td, 'ext2.py') - with open(ext2, 'w') as f: + with open(ext2, 'w', encoding='utf-8') as f: f.write(ext2_content) with prepended_to_syspath(td): @@ -77,7 +77,7 @@ def test_extension_builtins(): em = get_ipython().extension_manager with TemporaryDirectory() as td: ext3 = os.path.join(td, 'ext3.py') - with open(ext3, 'w') as f: + with open(ext3, 'w', encoding='utf-8') as f: f.write(ext3_content) assert 'ext3' not in em.loaded diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py index 41f5a358ea1..e86c4fcd560 100644 --- a/IPython/core/tests/test_interactiveshell.py +++ b/IPython/core/tests/test_interactiveshell.py @@ -485,11 +485,11 @@ def mock_print_func(value, sep=" ", end="\n", file=sys.stdout, flush=False): def test_mktempfile(self): filename = ip.mktempfile() # Check that we can open the file again on Windows - with open(filename, 'w') as f: + with open(filename, 'w', encoding='utf-8') as f: f.write('abc') filename = ip.mktempfile(data='blah') - with open(filename, 'r') as f: + with open(filename, 'r', encoding='utf-8') as f: self.assertEqual(f.read(), 'blah') def test_new_main_mod(self): @@ -545,7 +545,7 @@ def setUp(self): self.BASETESTDIR = tempfile.mkdtemp() self.TESTDIR = join(self.BASETESTDIR, u"åäö") os.mkdir(self.TESTDIR) - with open(join(self.TESTDIR, u"åäötestscript.py"), "w") as sfile: + with open(join(self.TESTDIR, u"åäötestscript.py"), "w", encoding='utf-8') as sfile: sfile.write("pass\n") self.oldpath = os.getcwd() os.chdir(self.TESTDIR) diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index 5294f82c51c..f645daf91b0 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -866,7 +866,7 @@ def test_file(): 'line1', 'line2', ])) - s = Path(fname).read_text() + s = Path(fname).read_text(encoding='utf-8') assert "line1\n" in s assert "line2" in s @@ -881,7 +881,7 @@ def test_file_single_quote(): 'line1', 'line2', ])) - s = Path(fname).read_text() + s = Path(fname).read_text(encoding='utf-8') assert "line1\n" in s assert "line2" in s @@ -896,7 +896,7 @@ def test_file_double_quote(): 'line1', 'line2', ])) - s = Path(fname).read_text() + s = Path(fname).read_text(encoding='utf-8') assert "line1\n" in s assert "line2" in s @@ -911,7 +911,7 @@ def test_file_var_expand(): 'line1', 'line2', ])) - s = Path(fname).read_text() + s = Path(fname).read_text(encoding='utf-8') assert "line1\n" in s assert "line2" in s @@ -944,7 +944,7 @@ def test_file_amend(): 'line3', 'line4', ])) - s = Path(fname).read_text() + s = Path(fname).read_text(encoding='utf-8') assert "line1\n" in s assert "line3\n" in s @@ -958,7 +958,7 @@ def test_file_spaces(): 'line1', 'line2', ])) - s = Path(fname).read_text() + s = Path(fname).read_text(encoding='utf-8') assert "line1\n" in s assert "line2" in s @@ -1154,11 +1154,11 @@ def test_save(): with TemporaryDirectory() as tmpdir: file = os.path.join(tmpdir, "testsave.py") ip.run_line_magic("save", "%s 1-10" % file) - content = Path(file).read_text() + content = Path(file).read_text(encoding='utf-8') assert content.count(cmds[0]) == 1 assert "coding: utf-8" in content ip.run_line_magic("save", "-a %s 1-10" % file) - content = Path(file).read_text() + content = Path(file).read_text(encoding='utf-8') assert content.count(cmds[0]) == 2 assert "coding: utf-8" in content @@ -1173,7 +1173,7 @@ def test_save_with_no_args(): with TemporaryDirectory() as tmpdir: path = os.path.join(tmpdir, "testsave.py") ip.run_line_magic("save", path) - content = Path(path).read_text() + content = Path(path).read_text(encoding='utf-8') expected_content = dedent( """\ # coding: utf-8 @@ -1336,7 +1336,7 @@ def test_run_module_from_import_hook(): "Test that a module can be loaded via an import hook" with TemporaryDirectory() as tmpdir: fullpath = os.path.join(tmpdir, 'my_tmp.py') - Path(fullpath).write_text(TEST_MODULE) + Path(fullpath).write_text(TEST_MODULE, encoding='utf-8') import importlib.abc import importlib.util @@ -1352,7 +1352,7 @@ def get_filename(self, fullname): def get_data(self, path): assert Path(path).samefile(fullpath) - return Path(fullpath).read_text() + return Path(fullpath).read_text(encoding='utf-8') sys.meta_path.insert(0, MyTempImporter()) diff --git a/IPython/core/tests/test_profile.py b/IPython/core/tests/test_profile.py index 8dd58cc67f4..22966e48ac3 100644 --- a/IPython/core/tests/test_profile.py +++ b/IPython/core/tests/test_profile.py @@ -84,10 +84,10 @@ def tearDown(self): def init(self, startup_file, startup, test): # write startup python file - with open(Path(self.pd.startup_dir) / startup_file, "w") as f: + with open(Path(self.pd.startup_dir) / startup_file, "w", encoding='utf-8') as f: f.write(startup) # write simple test file, to check that the startup file was run - with open(self.fname, 'w') as f: + with open(self.fname, 'w', encoding='utf-8') as f: f.write(test) def validate(self, output): @@ -111,7 +111,7 @@ def test_list_profiles_in(): if dec.unicode_paths: Path(td / u"profile_ünicode").mkdir(parents=True) - with open(td / "profile_file", "w") as f: + with open(td / "profile_file", "w", encoding='utf-8') as f: f.write("I am not a profile directory") profiles = list_profiles_in(td) diff --git a/IPython/core/tests/test_run.py b/IPython/core/tests/test_run.py index 0f73a781e3e..69576577e9e 100644 --- a/IPython/core/tests/test_run.py +++ b/IPython/core/tests/test_run.py @@ -64,7 +64,7 @@ def doctest_run_builtins(): In [3]: fname = tempfile.mkstemp('.py')[1] - In [3]: f = open(fname,'w') + In [3]: f = open(fname, 'w', encoding='utf-8') In [4]: dummy= f.write('pass\n') @@ -443,7 +443,7 @@ def writefile(self, name, content): d = os.path.dirname(path) if not os.path.isdir(d): os.makedirs(d) - with open(path, 'w') as f: + with open(path, 'w', encoding='utf-8') as f: f.write(textwrap.dedent(content)) def setUp(self): @@ -527,7 +527,7 @@ def test_module_options_with_separator(self): def test_run__name__(): with TemporaryDirectory() as td: path = pjoin(td, 'foo.py') - with open(path, 'w') as f: + with open(path, 'w', encoding='utf-8') as f: f.write("q = __name__") _ip.user_ns.pop("q", None) @@ -548,7 +548,7 @@ def test_run_tb(): """Test traceback offset in %run""" with TemporaryDirectory() as td: path = pjoin(td, 'foo.py') - with open(path, 'w') as f: + with open(path, 'w', encoding='utf-8') as f: f.write('\n'.join([ "def foo():", " return bar()", @@ -578,7 +578,7 @@ def test_multiprocessing_run(): sys.modules['__mp_main__'] = None try: path = pjoin(td, 'test.py') - with open(path, 'w') as f: + with open(path, 'w', encoding='utf-8') as f: f.write("import multiprocessing\nprint('hoy')") with capture_output() as io: _ip.run_line_magic('run', path) @@ -598,7 +598,7 @@ def test_script_tb(): """Test traceback offset in `ipython script.py`""" with TemporaryDirectory() as td: path = pjoin(td, 'foo.py') - with open(path, 'w') as f: + with open(path, 'w', encoding='utf-8') as f: f.write('\n'.join([ "def foo():", " return bar()", diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index 423998019b5..6e855d10186 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -58,7 +58,7 @@ def test_changing_py_file(self): """ with TemporaryDirectory() as td: fname = os.path.join(td, "foo.py") - with open(fname, "w") as f: + with open(fname, "w", encoding='utf-8') as f: f.write(file_1) with prepended_to_syspath(td): @@ -68,7 +68,7 @@ def test_changing_py_file(self): ip.run_cell("foo.f()") # Make the file shorter, so the line of the error is missing. - with open(fname, "w") as f: + with open(fname, "w", encoding='utf-8') as f: f.write(file_2) # For some reason, this was failing on the *second* call after @@ -92,7 +92,7 @@ def test_nonascii_path(self): # Non-ascii directory name as well. with TemporaryDirectory(suffix=u'é') as td: fname = os.path.join(td, u"fooé.py") - with open(fname, "w") as f: + with open(fname, "w", encoding='utf-8') as f: f.write(file_1) with prepended_to_syspath(td): @@ -172,7 +172,7 @@ def test_indentationerror_shows_line(self): with TemporaryDirectory() as td: fname = os.path.join(td, "foo.py") - with open(fname, "w") as f: + with open(fname, "w", encoding='utf-8') as f: f.write(indentationerror_file) with tt.AssertPrints("IndentationError"): @@ -221,14 +221,14 @@ def bar(): def test_changing_py_file(self): with TemporaryDirectory() as td: fname = os.path.join(td, "foo.py") - with open(fname, 'w') as f: + with open(fname, 'w', encoding='utf-8') as f: f.write(se_file_1) with tt.AssertPrints(["7/", "SyntaxError"]): ip.magic("run " + fname) # Modify the file - with open(fname, 'w') as f: + with open(fname, 'w', encoding='utf-8') as f: f.write(se_file_2) # The SyntaxError should point to the correct line diff --git a/IPython/extensions/storemagic.py b/IPython/extensions/storemagic.py index b4a8cf82442..ff124ee38bd 100644 --- a/IPython/extensions/storemagic.py +++ b/IPython/extensions/storemagic.py @@ -179,9 +179,9 @@ def store(self, parameter_s=''): if len(args) > 1 and args[1].startswith('>'): fnam = os.path.expanduser(args[1].lstrip('>').lstrip()) if args[1].startswith('>>'): - fil = open(fnam, 'a') + fil = open(fnam, 'a', encoding='utf-8') else: - fil = open(fnam, 'w') + fil = open(fnam, 'w', encoding='utf-8') with fil: obj = ip.ev(args[0]) print("Writing '%s' (%s) to file '%s'." % (args[0], diff --git a/IPython/extensions/tests/test_autoreload.py b/IPython/extensions/tests/test_autoreload.py index 65eb4a7b348..63bae802524 100644 --- a/IPython/extensions/tests/test_autoreload.py +++ b/IPython/extensions/tests/test_autoreload.py @@ -119,13 +119,13 @@ def write_file(self, filename, content): time.sleep(1.05) # Write - with open(filename, "w") as f: + with open(filename, "w", encoding='utf-8') as f: f.write(content) def new_module(self, code): code = textwrap.dedent(code) mod_name, mod_fn = self.get_module() - with open(mod_fn, "w") as f: + with open(mod_fn, "w", encoding='utf-8') as f: f.write(code) return mod_name, mod_fn diff --git a/IPython/lib/demo.py b/IPython/lib/demo.py index 14af5161cc8..0c96de6e72d 100644 --- a/IPython/lib/demo.py +++ b/IPython/lib/demo.py @@ -405,7 +405,7 @@ def edit(self,index=None): filename = self.shell.mktempfile(self.src_blocks[index]) self.shell.hooks.editor(filename, 1) - with open(Path(filename), "r") as f: + with open(Path(filename), "r", encoding='utf-8') as f: new_block = f.read() # update the source and colored block self.src_blocks[index] = new_block diff --git a/IPython/lib/tests/test_deepreload.py b/IPython/lib/tests/test_deepreload.py index 9759a7f69f7..f86e1027f98 100644 --- a/IPython/lib/tests/test_deepreload.py +++ b/IPython/lib/tests/test_deepreload.py @@ -19,9 +19,9 @@ def test_deepreload(): with TemporaryDirectory() as tmpdir: with prepended_to_syspath(tmpdir): tmpdirpath = Path(tmpdir) - with open(tmpdirpath / "A.py", "w") as f: + with open(tmpdirpath / "A.py", "w", encoding='utf-8') as f: f.write("class Object:\n pass\nok = True\n") - with open(tmpdirpath / "B.py", "w") as f: + with open(tmpdirpath / "B.py", "w", encoding='utf-8') as f: f.write("import A\nassert A.ok, 'we are fine'\n") import A import B diff --git a/IPython/testing/tests/test_tools.py b/IPython/testing/tests/test_tools.py index f9f7e716cbd..0e66ca39a99 100644 --- a/IPython/testing/tests/test_tools.py +++ b/IPython/testing/tests/test_tools.py @@ -62,7 +62,7 @@ def test_temp_pyfile(): src = 'pass\n' fname = tt.temp_pyfile(src) assert os.path.isfile(fname) - with open(fname) as fh2: + with open(fname, encoding='utf-8') as fh2: src2 = fh2.read() assert src2 == src diff --git a/IPython/testing/tools.py b/IPython/testing/tools.py index 0d52d88d2bf..fafb73fb8f7 100644 --- a/IPython/testing/tools.py +++ b/IPython/testing/tools.py @@ -428,7 +428,7 @@ def mute_warn(): def make_tempfile(name): """ Create an empty, named, temporary file for the duration of the context. """ - open(name, 'w').close() + open(name, 'w', encoding='utf-8').close() try: yield finally: diff --git a/IPython/tests/cve.py b/IPython/tests/cve.py index 9e0f6df715d..aba6e20fa79 100644 --- a/IPython/tests/cve.py +++ b/IPython/tests/cve.py @@ -33,7 +33,7 @@ def test_cve_2022_21699(): with TemporaryWorkingDirectory() as t: dangerous_startup_dir.mkdir(parents=True) - (dangerous_startup_dir / "foo.py").write_text(f'print("{dangerous_expected}")') + (dangerous_startup_dir / "foo.py").write_text(f'print("{dangerous_expected}")', encoding='utf-8') # 1 sec to make sure FS is flushed. # time.sleep(1) cmd = [sys.executable, "-m", "IPython"] diff --git a/IPython/utils/io.py b/IPython/utils/io.py index 69e4d4e0cdb..ef1be80988f 100644 --- a/IPython/utils/io.py +++ b/IPython/utils/io.py @@ -20,7 +20,7 @@ from .capture import CapturedIO, capture_output # setup stdin/stdout/stderr to sys.stdin/sys.stdout/sys.stderr -devnull = open(os.devnull, 'w') +devnull = open(os.devnull, 'w', encoding='utf-8') atexit.register(devnull.close) @@ -52,7 +52,8 @@ def __init__(self, file_or_name, mode="w", channel='stdout'): if hasattr(file_or_name, 'write') and hasattr(file_or_name, 'seek'): self.file = file_or_name else: - self.file = open(file_or_name, mode) + encoding = None if 'b' in mode else 'utf-8' + self.file = open(file_or_name, mode, encoding=encoding) self.channel = channel self.ostream = getattr(sys, channel) setattr(sys, channel, self) @@ -131,7 +132,7 @@ def temp_pyfile(src, ext='.py'): It is the caller's responsibility to close the open file and unlink it. """ fname = tempfile.mkstemp(ext)[1] - with open(Path(fname), "w") as f: + with open(Path(fname), "w", encoding='utf-8') as f: f.write(src) f.flush() return fname diff --git a/IPython/utils/tempdir.py b/IPython/utils/tempdir.py index bbfffe9c923..0729a8417dc 100644 --- a/IPython/utils/tempdir.py +++ b/IPython/utils/tempdir.py @@ -24,7 +24,8 @@ def __init__(self, filename, mode='w+b', bufsize=-1, **kwds): """ self._tmpdir = TemporaryDirectory(**kwds) path = Path(self._tmpdir.name) / filename - self.file = open(path, mode, bufsize) + encoding = None if 'b' in mode else 'utf-8' + self.file = open(path, mode, bufsize, encoding=encoding) def cleanup(self): self.file.close() diff --git a/IPython/utils/tests/test_module_paths.py b/IPython/utils/tests/test_module_paths.py index 12679990c0c..2abf889514c 100644 --- a/IPython/utils/tests/test_module_paths.py +++ b/IPython/utils/tests/test_module_paths.py @@ -32,7 +32,7 @@ old_syspath = sys.path def make_empty_file(fname): - open(fname, 'w').close() + open(fname, 'w', encoding='utf-8').close() def setup_module(): diff --git a/IPython/utils/tests/test_path.py b/IPython/utils/tests/test_path.py index b27e4355383..f92e6ae1f6c 100644 --- a/IPython/utils/tests/test_path.py +++ b/IPython/utils/tests/test_path.py @@ -295,7 +295,7 @@ def test_not_writable_ipdir(self): ipdir = os.path.join(tmpdir, '.ipython') os.mkdir(ipdir, 0o555) try: - open(os.path.join(ipdir, "_foo_"), 'w').close() + open(os.path.join(ipdir, "_foo_"), 'w', encoding='utf-8').close() except IOError: pass else: @@ -352,7 +352,7 @@ def setUpClass(cls): with cls.in_tempdir(): # Create empty files for fname in cls.filenames: - open(os.path.join(td, fname), 'w').close() + open(os.path.join(td, fname), 'w', encoding='utf-8').close() @classmethod def tearDownClass(cls): @@ -428,7 +428,7 @@ def test_ensure_dir_exists(): assert os.path.isdir(d) path.ensure_dir_exists(d) # no-op f = os.path.join(td, 'ƒile') - open(f, 'w').close() # touch + open(f, 'w', encoding='utf-8').close() # touch with pytest.raises(IOError): path.ensure_dir_exists(f) @@ -436,7 +436,7 @@ class TestLinkOrCopy(unittest.TestCase): def setUp(self): self.tempdir = TemporaryDirectory() self.src = self.dst("src") - with open(self.src, "w") as f: + with open(self.src, "w", encoding='utf-8') as f: f.write("Hello, world!") def tearDown(self): @@ -456,8 +456,8 @@ def assert_inode_equal(self, a, b): ), "%r and %r do not reference the same indoes" % (a, b) def assert_content_equal(self, a, b): - with open(a) as a_f: - with open(b) as b_f: + with open(a, 'rb') as a_f: + with open(b, 'rb') as b_f: assert a_f.read() == b_f.read() @skip_win32 @@ -477,7 +477,7 @@ def test_link_into_dir(self): @skip_win32 def test_target_exists(self): dst = self.dst("target") - open(dst, "w").close() + open(dst, "w", encoding='utf-8').close() path.link_or_copy(self.src, dst) self.assert_inode_equal(self.src, dst) diff --git a/IPython/utils/tests/test_pycolorize.py b/IPython/utils/tests/test_pycolorize.py index 05d51b6e44d..986b9178800 100644 --- a/IPython/utils/tests/test_pycolorize.py +++ b/IPython/utils/tests/test_pycolorize.py @@ -39,7 +39,7 @@ def function(arg, *args, kwarg=True, **kwargs): pass is True False == None - with io.open(ru'unicode'): + with io.open(ru'unicode', encoding='utf-8'): raise ValueError("\n escape \r sequence") print("wěird ünicoðe") diff --git a/docs/autogen_config.py b/docs/autogen_config.py index 98d13d3b0cb..56733f3b422 100755 --- a/docs/autogen_config.py +++ b/docs/autogen_config.py @@ -102,7 +102,7 @@ def reverse_aliases(app): def write_doc(name, title, app, preamble=None): trait_aliases = reverse_aliases(app) filename = options / (name + ".rst") - with open(filename, "w") as f: + with open(filename, "w", encoding='utf-8') as f: f.write(title + "\n") f.write(("=" * len(title)) + "\n") f.write("\n") @@ -117,7 +117,7 @@ def write_doc(name, title, app, preamble=None): if __name__ == '__main__': # Touch this file for the make target - Path(generated).write_text("") + Path(generated).write_text("", encoding='utf-8') write_doc('terminal', 'Terminal IPython options', TerminalIPythonApp()) write_doc('kernel', 'IPython kernel options', IPKernelApp(), diff --git a/docs/autogen_magics.py b/docs/autogen_magics.py index 1f8a47ef506..bfeb6aa7733 100644 --- a/docs/autogen_magics.py +++ b/docs/autogen_magics.py @@ -63,4 +63,4 @@ def sortkey(s): return s[0].lower() src_path = Path(__file__).parent dest = src_path.joinpath("source", "interactive", "magics-generated.txt") -dest.write_text("\n".join(output)) +dest.write_text("\n".join(output), encoding='utf-8') diff --git a/docs/autogen_shortcuts.py b/docs/autogen_shortcuts.py index 802fa03e100..7ee0f910768 100755 --- a/docs/autogen_shortcuts.py +++ b/docs/autogen_shortcuts.py @@ -89,6 +89,6 @@ def sort_key(item): (single_filter, "single_filtered"), (multi_filter, "multi_filtered"), ]: - with (dest / "{}.csv".format(output_filename)).open("w") as csv: + with (dest / "{}.csv".format(output_filename)).open("w", encoding='utf-8') as csv: for (shortcut, flt), v in sorted(filters.items(), key=sort_key): csv.write(":kbd:`{}`\t{}\t{}\n".format(shortcut, flt, v)) diff --git a/docs/source/conf.py b/docs/source/conf.py index 2312cc248b0..ba9aa1dbc88 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,7 +28,7 @@ for name in ("config", "api", "magics", "shortcuts"): fname = Path("autogen_{}.py".format(name)) fpath = (Path(__file__).parent).joinpath("..", fname) - with open(fpath) as f: + with open(fpath, encoding='utf-8') as f: exec(compile(f.read(), fname, 'exec'), { '__file__': fpath, '__name__': '__main__', @@ -45,7 +45,7 @@ # We load the ipython release info into a dict by explicit execution iprelease = {} -exec(compile(open('../../IPython/core/release.py').read(), '../../IPython/core/release.py', 'exec'),iprelease) +exec(compile(open('../../IPython/core/release.py', encoding='utf-8').read(), '../../IPython/core/release.py', 'exec'),iprelease) # General configuration # --------------------- diff --git a/docs/sphinxext/apigen.py b/docs/sphinxext/apigen.py index 5d352c7a22f..71a384b7d2a 100644 --- a/docs/sphinxext/apigen.py +++ b/docs/sphinxext/apigen.py @@ -392,7 +392,7 @@ def write_modules_api(self, modules,outdir): # write out to file outfile = os.path.join(outdir, m + self.rst_extension) - with open(outfile, 'wt') as fileobj: + with open(outfile, 'wt', encoding='utf-8') as fileobj: fileobj.write(api_str) written_modules.append(m) self.written_modules = written_modules @@ -444,7 +444,7 @@ def write_index(self, outdir, path='gen.rst', relative_to=None): relpath = outdir.replace(relative_to + os.path.sep, '') else: relpath = outdir - with open(path,'wt') as idx: + with open(path,'wt', encoding='utf-8') as idx: w = idx.write w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n') w('.. autosummary::\n' diff --git a/examples/IPython Kernel/ipython-get-history.py b/examples/IPython Kernel/ipython-get-history.py index a2d2ab7b0ab..fe38081fbad 100755 --- a/examples/IPython Kernel/ipython-get-history.py +++ b/examples/IPython Kernel/ipython-get-history.py @@ -24,7 +24,7 @@ session_number = int(sys.argv[1]) if len(sys.argv) > 2: filepath = Path(sys.argv[2]) - dest = open(filepath, "w") + dest = open(filepath, "w", encoding='utf-8') raw = not filepath.name.endswith(".py") else: dest = sys.stdout diff --git a/setupbase.py b/setupbase.py index ecea9e788cc..788ce2491fb 100644 --- a/setupbase.py +++ b/setupbase.py @@ -36,7 +36,7 @@ def execfile(fname, globs, locs=None): locs = locs or globs - with open(fname) as f: + with open(fname, encoding='utf-8') as f: exec(compile(f.read(), fname, "exec"), globs, locs) # A little utility we'll need below, since glob() does NOT allow you to do @@ -336,7 +336,7 @@ def _record_commit(self, base_dir): os.remove(out_pth) except (IOError, OSError): pass - with open(out_pth, 'w') as out_file: + with open(out_pth, 'w', encoding='utf-8') as out_file: out_file.writelines([ '# GENERATED BY setup.py\n', 'commit = u"%s"\n' % repo_commit, diff --git a/tools/fixup_whats_new_pr.py b/tools/fixup_whats_new_pr.py index a2741ef9ebf..397c0bf38e3 100644 --- a/tools/fixup_whats_new_pr.py +++ b/tools/fixup_whats_new_pr.py @@ -22,14 +22,14 @@ def main(): print("Adding pseudo-title to:", filepath.name) title = filepath.name[:-4].split("/")[-1].replace("-", " ").capitalize() - data = filepath.read_text() + data = filepath.read_text(encoding='utf-8') try: if data and data.splitlines()[1].startswith('='): continue except IndexError: pass - with filepath.open("w") as f: + with filepath.open("w", encoding='utf-8') as f: f.write(title + "\n") f.write("=" * len(title) + "\n\n") f.write(data) diff --git a/tools/toollib.py b/tools/toollib.py index 90727f0acda..c77bbbc6853 100644 --- a/tools/toollib.py +++ b/tools/toollib.py @@ -45,4 +45,4 @@ def get_ipdir(): def execfile(fname, globs, locs=None): locs = locs or globs - exec(compile(open(fname).read(), fname, "exec"), globs, locs) + exec(compile(open(fname, encoding='utf-8').read(), fname, "exec"), globs, locs) From 1a9d9554bcee466394990535e190d55008904df8 Mon Sep 17 00:00:00 2001 From: gousaiyang Date: Fri, 4 Feb 2022 22:52:25 -0800 Subject: [PATCH 50/85] Format code --- IPython/core/application.py | 2 +- IPython/core/crashhandler.py | 2 +- IPython/core/display.py | 7 +- IPython/core/interactiveshell.py | 8 +- IPython/core/magics/code.py | 12 +- IPython/core/magics/execution.py | 2 +- IPython/core/magics/packaging.py | 2 +- IPython/core/page.py | 19 +-- IPython/core/tests/test_application.py | 4 +- IPython/core/tests/test_completer.py | 6 +- IPython/core/tests/test_completerlib.py | 8 +- IPython/core/tests/test_extension.py | 14 +- IPython/core/tests/test_interactiveshell.py | 14 +- IPython/core/tests/test_magic.py | 132 ++++++++++++------ IPython/core/tests/test_profile.py | 6 +- IPython/core/tests/test_run.py | 54 ++++--- IPython/core/tests/test_ultratb.py | 12 +- IPython/extensions/storemagic.py | 10 +- IPython/extensions/tests/test_autoreload.py | 4 +- IPython/lib/demo.py | 2 +- IPython/lib/tests/test_deepreload.py | 4 +- IPython/testing/tests/test_tools.py | 2 +- IPython/testing/tools.py | 5 +- IPython/tests/cve.py | 4 +- IPython/utils/io.py | 6 +- IPython/utils/tempdir.py | 2 +- IPython/utils/tests/test_module_paths.py | 2 +- IPython/utils/tests/test_path.py | 18 +-- docs/autogen_config.py | 4 +- docs/autogen_magics.py | 2 +- docs/autogen_shortcuts.py | 4 +- docs/source/conf.py | 22 ++- docs/sphinxext/apigen.py | 7 +- .../IPython Kernel/ipython-get-history.py | 2 +- setupbase.py | 15 +- tools/fixup_whats_new_pr.py | 4 +- tools/toollib.py | 2 +- 37 files changed, 249 insertions(+), 176 deletions(-) diff --git a/IPython/core/application.py b/IPython/core/application.py index 90ecf13179d..0cdea5c69b8 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -470,7 +470,7 @@ def stage_default_config_file(self): config_file = Path(self.profile_dir.location) / self.config_file_name if self.overwrite or not config_file.exists(): self.log.warning("Generating default config file: %r" % (config_file)) - config_file.write_text(s, encoding='utf-8') + config_file.write_text(s, encoding="utf-8") @catch_config_error def initialize(self, argv=None): diff --git a/IPython/core/crashhandler.py b/IPython/core/crashhandler.py index 3ee4879e4e8..4af39361e80 100644 --- a/IPython/core/crashhandler.py +++ b/IPython/core/crashhandler.py @@ -185,7 +185,7 @@ def __call__(self, etype, evalue, etb): # and generate a complete report on disk try: - report = open(report_name, 'w', encoding='utf-8') + report = open(report_name, "w", encoding="utf-8") except: print('Could not create crash report on disk.', file=sys.stderr) return diff --git a/IPython/core/display.py b/IPython/core/display.py index 608188ccece..d36a176c3bf 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -349,7 +349,7 @@ def _data_and_metadata(self): def reload(self): """Reload the raw data from file or URL.""" if self.filename is not None: - encoding = None if 'b' in self._read_flags else 'utf-8' + encoding = None if "b" in self._read_flags else "utf-8" with open(self.filename, self._read_flags, encoding=encoding) as f: self.data = f.read() elif self.url is not None: @@ -370,8 +370,11 @@ def reload(self): if 'gzip' in response.headers['content-encoding']: import gzip from io import BytesIO + # assume utf-8 if encoding is not specified - with gzip.open(BytesIO(data), 'rt', encoding=encoding or 'utf-8') as fp: + with gzip.open( + BytesIO(data), "rt", encoding=encoding or "utf-8" + ) as fp: encoding = None data = fp.read() diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 65c3635f2fe..4f3105a2ebb 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2626,7 +2626,7 @@ def safe_execfile(self, fname, *where, exit_ignore=False, raise_exceptions=False # Make sure we can open the file try: - with fname.open('rb'): + with fname.open("rb"): pass except: warn('Could not open file <%s> for safe execution.' % fname) @@ -2684,7 +2684,7 @@ def safe_execfile_ipy(self, fname, shell_futures=False, raise_exceptions=False): # Make sure we can open the file try: - with fname.open('rb'): + with fname.open("rb"): pass except: warn('Could not open file <%s> for safe execution.' % fname) @@ -2706,7 +2706,7 @@ def get_cells(): if cell.cell_type == 'code': yield cell.source else: - yield fname.read_text(encoding='utf-8') + yield fname.read_text(encoding="utf-8") with prepended_to_syspath(dname): try: @@ -3458,7 +3458,7 @@ def mktempfile(self, data=None, prefix='ipython_edit_'): self.tempfiles.append(file_path) if data: - file_path.write_text(data, encoding='utf-8') + file_path.write_text(data, encoding="utf-8") return filename def ask_yes_no(self, prompt, default=None, interrupt=None): diff --git a/IPython/core/magics/code.py b/IPython/core/magics/code.py index 6f44018ff72..65ba52b8bbf 100644 --- a/IPython/core/magics/code.py +++ b/IPython/core/magics/code.py @@ -538,7 +538,7 @@ def _edit_macro(self,mname,macro): self.shell.hooks.editor(filename) # and make a new macro object, to replace the old one - mvalue = Path(filename).read_text(encoding='utf-8') + mvalue = Path(filename).read_text(encoding="utf-8") self.shell.user_ns[mname] = Macro(mvalue) @skip_doctest @@ -728,7 +728,7 @@ def edit(self, parameter_s='',last_call=['','']): # XXX TODO: should this be generalized for all string vars? # For now, this is special-cased to blocks created by cpaste if args.strip() == "pasted_block": - self.shell.user_ns["pasted_block"] = filepath.read_text(encoding='utf-8') + self.shell.user_ns["pasted_block"] = filepath.read_text(encoding="utf-8") if 'x' in opts: # -x prevents actual execution print() @@ -736,9 +736,9 @@ def edit(self, parameter_s='',last_call=['','']): print('done. Executing edited code...') with preserve_keys(self.shell.user_ns, '__file__'): if not is_temp: - self.shell.user_ns['__file__'] = filename - if 'r' in opts: # Untranslated IPython code - source = filepath.read_text(encoding='utf-8') + self.shell.user_ns["__file__"] = filename + if "r" in opts: # Untranslated IPython code + source = filepath.read_text(encoding="utf-8") self.shell.run_cell(source, store_history=False) else: self.shell.safe_execfile(filename, self.shell.user_ns, @@ -746,7 +746,7 @@ def edit(self, parameter_s='',last_call=['','']): if is_temp: try: - return filepath.read_text(encoding='utf-8') + return filepath.read_text(encoding="utf-8") except IOError as msg: if Path(msg.filename) == filepath: warn('File not found. Did you forget to save?') diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py index a90bd4efac0..371da5b1f98 100644 --- a/IPython/core/magics/execution.py +++ b/IPython/core/magics/execution.py @@ -360,7 +360,7 @@ def _run_with_profiler(self, code, opts, namespace): if text_file: pfile = Path(text_file) pfile.touch(exist_ok=True) - pfile.write_text(output, encoding='utf-8') + pfile.write_text(output, encoding="utf-8") print( f"\n*** Profile printout saved to text file {repr(text_file)}.{sys_exit}" diff --git a/IPython/core/magics/packaging.py b/IPython/core/magics/packaging.py index 1859a942929..2f7652c169b 100644 --- a/IPython/core/magics/packaging.py +++ b/IPython/core/magics/packaging.py @@ -32,7 +32,7 @@ def _get_conda_executable(): # Otherwise, attempt to extract the executable from conda history. # This applies in any conda environment. - history = Path(sys.prefix, "conda-meta", "history").read_text(encoding='utf-8') + history = Path(sys.prefix, "conda-meta", "history").read_text(encoding="utf-8") match = re.search( r"^#\s*cmd:\s*(?P.*conda)\s[create|install]", history, diff --git a/IPython/core/page.py b/IPython/core/page.py index 7ebb6ec0047..d3e6a9eef50 100644 --- a/IPython/core/page.py +++ b/IPython/core/page.py @@ -199,7 +199,7 @@ def pager_page(strng, start=0, screen_lines=0, pager_cmd=None): tmppath = Path(tmpname) try: os.close(fd) - with tmppath.open("wt", encoding='utf-8') as tmpfile: + with tmppath.open("wt", encoding="utf-8") as tmpfile: tmpfile.write(strng) cmd = "%s < %s" % (pager_cmd, tmppath) # tmpfile needs to be closed for windows @@ -213,12 +213,15 @@ def pager_page(strng, start=0, screen_lines=0, pager_cmd=None): try: retval = None # Emulate os.popen, but redirect stderr - proc = subprocess.Popen(pager_cmd, - shell=True, - stdin=subprocess.PIPE, - stderr=subprocess.DEVNULL - ) - pager = os._wrap_close(io.TextIOWrapper(proc.stdin, encoding='utf-8'), proc) + proc = subprocess.Popen( + pager_cmd, + shell=True, + stdin=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + pager = os._wrap_close( + io.TextIOWrapper(proc.stdin, encoding="utf-8"), proc + ) try: pager_encoding = pager.encoding or sys.stdout.encoding pager.write(strng) @@ -277,7 +280,7 @@ def page_file(fname, start=0, pager_cmd=None): try: if start > 0: start -= 1 - page(open(fname, encoding='utf-8').read(),start) + page(open(fname, encoding="utf-8").read(), start) except: print('Unable to show file',repr(fname)) diff --git a/IPython/core/tests/test_application.py b/IPython/core/tests/test_application.py index f4b11b5a928..74ea0f79006 100644 --- a/IPython/core/tests/test_application.py +++ b/IPython/core/tests/test_application.py @@ -34,7 +34,7 @@ def test_unicode_ipdir(): ipdir = tempfile.mkdtemp(suffix=u"€") # Create the config file, so it tries to load it. - with open(os.path.join(ipdir, 'ipython_config.py'), "w", encoding='utf-8') as f: + with open(os.path.join(ipdir, "ipython_config.py"), "w", encoding="utf-8") as f: pass old_ipdir1 = os.environ.pop("IPYTHONDIR", None) @@ -59,7 +59,7 @@ class TestApp(BaseIPythonApplication): test = Unicode().tag(config=True) # Create the config file, so it tries to load it. - with open(os.path.join(td, 'ipython_config.py'), "w", encoding='utf-8') as f: + with open(os.path.join(td, "ipython_config.py"), "w", encoding="utf-8") as f: f.write("c.TestApp.test = 'config file'") app = TestApp() diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 7ce160a5aec..5f791e84f83 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -346,7 +346,7 @@ def test_abspath_file_completions(self): suffixes = ["1", "2"] names = [prefix + s for s in suffixes] for n in names: - open(n, "w", encoding='utf-8').close() + open(n, "w", encoding="utf-8").close() # Check simple completion c = ip.complete(prefix)[1] @@ -365,7 +365,7 @@ def test_local_file_completions(self): suffixes = ["1", "2"] names = [prefix + s for s in suffixes] for n in names: - open(n, "w", encoding='utf-8').close() + open(n, "w", encoding="utf-8").close() # Check simple completion c = ip.complete(prefix)[1] @@ -381,7 +381,7 @@ def test_quoted_file_completions(self): ip = get_ipython() with TemporaryWorkingDirectory(): name = "foo'bar" - open(name, "w", encoding='utf-8').close() + open(name, "w", encoding="utf-8").close() # Don't escape Windows escaped = name if sys.platform == "win32" else "foo\\'bar" diff --git a/IPython/core/tests/test_completerlib.py b/IPython/core/tests/test_completerlib.py index 23e24e6bfd6..fbbc258673f 100644 --- a/IPython/core/tests/test_completerlib.py +++ b/IPython/core/tests/test_completerlib.py @@ -33,7 +33,7 @@ class Test_magic_run_completer(unittest.TestCase): def setUp(self): self.BASETESTDIR = tempfile.mkdtemp() for fil in self.files: - with open(join(self.BASETESTDIR, fil), "w", encoding='utf-8') as sfile: + with open(join(self.BASETESTDIR, fil), "w", encoding="utf-8") as sfile: sfile.write("pass\n") for d in self.dirs: os.mkdir(join(self.BASETESTDIR, d)) @@ -89,7 +89,7 @@ class Test_magic_run_completer_nonascii(unittest.TestCase): def setUp(self): self.BASETESTDIR = tempfile.mkdtemp() for fil in [u"aaø.py", u"a.py", u"b.py"]: - with open(join(self.BASETESTDIR, fil), "w", encoding='utf-8') as sfile: + with open(join(self.BASETESTDIR, fil), "w", encoding="utf-8") as sfile: sfile.write("pass\n") self.oldpath = os.getcwd() os.chdir(self.BASETESTDIR) @@ -133,8 +133,8 @@ def test_import_invalid_module(): with TemporaryDirectory() as tmpdir: sys.path.insert( 0, tmpdir ) for name in invalid_module_names | valid_module_names: - filename = os.path.join(tmpdir, name + '.py') - open(filename, 'w', encoding='utf-8').close() + filename = os.path.join(tmpdir, name + ".py") + open(filename, "w", encoding="utf-8").close() s = set( module_completion('import foo') ) intersection = s.intersection(invalid_module_names) diff --git a/IPython/core/tests/test_extension.py b/IPython/core/tests/test_extension.py index 958fd2b4a2c..59e21dcacf2 100644 --- a/IPython/core/tests/test_extension.py +++ b/IPython/core/tests/test_extension.py @@ -26,12 +26,12 @@ def load_ipython_extension(ip): def test_extension_loading(): em = get_ipython().extension_manager with TemporaryDirectory() as td: - ext1 = os.path.join(td, 'ext1.py') - with open(ext1, 'w', encoding='utf-8') as f: + ext1 = os.path.join(td, "ext1.py") + with open(ext1, "w", encoding="utf-8") as f: f.write(ext1_content) - - ext2 = os.path.join(td, 'ext2.py') - with open(ext2, 'w', encoding='utf-8') as f: + + ext2 = os.path.join(td, "ext2.py") + with open(ext2, "w", encoding="utf-8") as f: f.write(ext2_content) with prepended_to_syspath(td): @@ -76,8 +76,8 @@ def test_extension_loading(): def test_extension_builtins(): em = get_ipython().extension_manager with TemporaryDirectory() as td: - ext3 = os.path.join(td, 'ext3.py') - with open(ext3, 'w', encoding='utf-8') as f: + ext3 = os.path.join(td, "ext3.py") + with open(ext3, "w", encoding="utf-8") as f: f.write(ext3_content) assert 'ext3' not in em.loaded diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py index e86c4fcd560..09dbd967706 100644 --- a/IPython/core/tests/test_interactiveshell.py +++ b/IPython/core/tests/test_interactiveshell.py @@ -485,12 +485,12 @@ def mock_print_func(value, sep=" ", end="\n", file=sys.stdout, flush=False): def test_mktempfile(self): filename = ip.mktempfile() # Check that we can open the file again on Windows - with open(filename, 'w', encoding='utf-8') as f: - f.write('abc') + with open(filename, "w", encoding="utf-8") as f: + f.write("abc") - filename = ip.mktempfile(data='blah') - with open(filename, 'r', encoding='utf-8') as f: - self.assertEqual(f.read(), 'blah') + filename = ip.mktempfile(data="blah") + with open(filename, "r", encoding="utf-8") as f: + self.assertEqual(f.read(), "blah") def test_new_main_mod(self): # Smoketest to check that this accepts a unicode module name @@ -545,7 +545,9 @@ def setUp(self): self.BASETESTDIR = tempfile.mkdtemp() self.TESTDIR = join(self.BASETESTDIR, u"åäö") os.mkdir(self.TESTDIR) - with open(join(self.TESTDIR, u"åäötestscript.py"), "w", encoding='utf-8') as sfile: + with open( + join(self.TESTDIR, u"åäötestscript.py"), "w", encoding="utf-8" + ) as sfile: sfile.write("pass\n") self.oldpath = os.getcwd() os.chdir(self.TESTDIR) diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index f645daf91b0..0f6d4aaa951 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -861,12 +861,18 @@ def test_file(): """Basic %%writefile""" ip = get_ipython() with TemporaryDirectory() as td: - fname = os.path.join(td, 'file1') - ip.run_cell_magic("writefile", fname, u'\n'.join([ - 'line1', - 'line2', - ])) - s = Path(fname).read_text(encoding='utf-8') + fname = os.path.join(td, "file1") + ip.run_cell_magic( + "writefile", + fname, + "\n".join( + [ + "line1", + "line2", + ] + ), + ) + s = Path(fname).read_text(encoding="utf-8") assert "line1\n" in s assert "line2" in s @@ -876,12 +882,18 @@ def test_file_single_quote(): """Basic %%writefile with embedded single quotes""" ip = get_ipython() with TemporaryDirectory() as td: - fname = os.path.join(td, '\'file1\'') - ip.run_cell_magic("writefile", fname, u'\n'.join([ - 'line1', - 'line2', - ])) - s = Path(fname).read_text(encoding='utf-8') + fname = os.path.join(td, "'file1'") + ip.run_cell_magic( + "writefile", + fname, + "\n".join( + [ + "line1", + "line2", + ] + ), + ) + s = Path(fname).read_text(encoding="utf-8") assert "line1\n" in s assert "line2" in s @@ -892,11 +904,17 @@ def test_file_double_quote(): ip = get_ipython() with TemporaryDirectory() as td: fname = os.path.join(td, '"file1"') - ip.run_cell_magic("writefile", fname, u'\n'.join([ - 'line1', - 'line2', - ])) - s = Path(fname).read_text(encoding='utf-8') + ip.run_cell_magic( + "writefile", + fname, + "\n".join( + [ + "line1", + "line2", + ] + ), + ) + s = Path(fname).read_text(encoding="utf-8") assert "line1\n" in s assert "line2" in s @@ -905,13 +923,19 @@ def test_file_var_expand(): """%%writefile $filename""" ip = get_ipython() with TemporaryDirectory() as td: - fname = os.path.join(td, 'file1') - ip.user_ns['filename'] = fname - ip.run_cell_magic("writefile", '$filename', u'\n'.join([ - 'line1', - 'line2', - ])) - s = Path(fname).read_text(encoding='utf-8') + fname = os.path.join(td, "file1") + ip.user_ns["filename"] = fname + ip.run_cell_magic( + "writefile", + "$filename", + "\n".join( + [ + "line1", + "line2", + ] + ), + ) + s = Path(fname).read_text(encoding="utf-8") assert "line1\n" in s assert "line2" in s @@ -935,16 +959,28 @@ def test_file_amend(): """%%writefile -a amends files""" ip = get_ipython() with TemporaryDirectory() as td: - fname = os.path.join(td, 'file2') - ip.run_cell_magic("writefile", fname, u'\n'.join([ - 'line1', - 'line2', - ])) - ip.run_cell_magic("writefile", "-a %s" % fname, u'\n'.join([ - 'line3', - 'line4', - ])) - s = Path(fname).read_text(encoding='utf-8') + fname = os.path.join(td, "file2") + ip.run_cell_magic( + "writefile", + fname, + "\n".join( + [ + "line1", + "line2", + ] + ), + ) + ip.run_cell_magic( + "writefile", + "-a %s" % fname, + "\n".join( + [ + "line3", + "line4", + ] + ), + ) + s = Path(fname).read_text(encoding="utf-8") assert "line1\n" in s assert "line3\n" in s @@ -954,11 +990,17 @@ def test_file_spaces(): ip = get_ipython() with TemporaryWorkingDirectory() as td: fname = "file name" - ip.run_cell_magic("file", '"%s"'%fname, u'\n'.join([ - 'line1', - 'line2', - ])) - s = Path(fname).read_text(encoding='utf-8') + ip.run_cell_magic( + "file", + '"%s"' % fname, + "\n".join( + [ + "line1", + "line2", + ] + ), + ) + s = Path(fname).read_text(encoding="utf-8") assert "line1\n" in s assert "line2" in s @@ -1154,11 +1196,11 @@ def test_save(): with TemporaryDirectory() as tmpdir: file = os.path.join(tmpdir, "testsave.py") ip.run_line_magic("save", "%s 1-10" % file) - content = Path(file).read_text(encoding='utf-8') + content = Path(file).read_text(encoding="utf-8") assert content.count(cmds[0]) == 1 assert "coding: utf-8" in content ip.run_line_magic("save", "-a %s 1-10" % file) - content = Path(file).read_text(encoding='utf-8') + content = Path(file).read_text(encoding="utf-8") assert content.count(cmds[0]) == 2 assert "coding: utf-8" in content @@ -1173,7 +1215,7 @@ def test_save_with_no_args(): with TemporaryDirectory() as tmpdir: path = os.path.join(tmpdir, "testsave.py") ip.run_line_magic("save", path) - content = Path(path).read_text(encoding='utf-8') + content = Path(path).read_text(encoding="utf-8") expected_content = dedent( """\ # coding: utf-8 @@ -1335,8 +1377,8 @@ def test_timeit_arguments(): def test_run_module_from_import_hook(): "Test that a module can be loaded via an import hook" with TemporaryDirectory() as tmpdir: - fullpath = os.path.join(tmpdir, 'my_tmp.py') - Path(fullpath).write_text(TEST_MODULE, encoding='utf-8') + fullpath = os.path.join(tmpdir, "my_tmp.py") + Path(fullpath).write_text(TEST_MODULE, encoding="utf-8") import importlib.abc import importlib.util @@ -1352,7 +1394,7 @@ def get_filename(self, fullname): def get_data(self, path): assert Path(path).samefile(fullpath) - return Path(fullpath).read_text(encoding='utf-8') + return Path(fullpath).read_text(encoding="utf-8") sys.meta_path.insert(0, MyTempImporter()) diff --git a/IPython/core/tests/test_profile.py b/IPython/core/tests/test_profile.py index 22966e48ac3..d034b50f219 100644 --- a/IPython/core/tests/test_profile.py +++ b/IPython/core/tests/test_profile.py @@ -84,10 +84,10 @@ def tearDown(self): def init(self, startup_file, startup, test): # write startup python file - with open(Path(self.pd.startup_dir) / startup_file, "w", encoding='utf-8') as f: + with open(Path(self.pd.startup_dir) / startup_file, "w", encoding="utf-8") as f: f.write(startup) # write simple test file, to check that the startup file was run - with open(self.fname, 'w', encoding='utf-8') as f: + with open(self.fname, "w", encoding="utf-8") as f: f.write(test) def validate(self, output): @@ -111,7 +111,7 @@ def test_list_profiles_in(): if dec.unicode_paths: Path(td / u"profile_ünicode").mkdir(parents=True) - with open(td / "profile_file", "w", encoding='utf-8') as f: + with open(td / "profile_file", "w", encoding="utf-8") as f: f.write("I am not a profile directory") profiles = list_profiles_in(td) diff --git a/IPython/core/tests/test_run.py b/IPython/core/tests/test_run.py index 69576577e9e..9204e81e8e8 100644 --- a/IPython/core/tests/test_run.py +++ b/IPython/core/tests/test_run.py @@ -443,7 +443,7 @@ def writefile(self, name, content): d = os.path.dirname(path) if not os.path.isdir(d): os.makedirs(d) - with open(path, 'w', encoding='utf-8') as f: + with open(path, "w", encoding="utf-8") as f: f.write(textwrap.dedent(content)) def setUp(self): @@ -526,8 +526,8 @@ def test_module_options_with_separator(self): def test_run__name__(): with TemporaryDirectory() as td: - path = pjoin(td, 'foo.py') - with open(path, 'w', encoding='utf-8') as f: + path = pjoin(td, "foo.py") + with open(path, "w", encoding="utf-8") as f: f.write("q = __name__") _ip.user_ns.pop("q", None) @@ -547,15 +547,19 @@ def test_run__name__(): def test_run_tb(): """Test traceback offset in %run""" with TemporaryDirectory() as td: - path = pjoin(td, 'foo.py') - with open(path, 'w', encoding='utf-8') as f: - f.write('\n'.join([ - "def foo():", - " return bar()", - "def bar():", - " raise RuntimeError('hello!')", - "foo()", - ])) + path = pjoin(td, "foo.py") + with open(path, "w", encoding="utf-8") as f: + f.write( + "\n".join( + [ + "def foo():", + " return bar()", + "def bar():", + " raise RuntimeError('hello!')", + "foo()", + ] + ) + ) with capture_output() as io: _ip.magic('run {}'.format(path)) out = io.stdout @@ -577,8 +581,8 @@ def test_multiprocessing_run(): mpm = sys.modules.get('__mp_main__') sys.modules['__mp_main__'] = None try: - path = pjoin(td, 'test.py') - with open(path, 'w', encoding='utf-8') as f: + path = pjoin(td, "test.py") + with open(path, "w", encoding="utf-8") as f: f.write("import multiprocessing\nprint('hoy')") with capture_output() as io: _ip.run_line_magic('run', path) @@ -597,15 +601,19 @@ def test_multiprocessing_run(): def test_script_tb(): """Test traceback offset in `ipython script.py`""" with TemporaryDirectory() as td: - path = pjoin(td, 'foo.py') - with open(path, 'w', encoding='utf-8') as f: - f.write('\n'.join([ - "def foo():", - " return bar()", - "def bar():", - " raise RuntimeError('hello!')", - "foo()", - ])) + path = pjoin(td, "foo.py") + with open(path, "w", encoding="utf-8") as f: + f.write( + "\n".join( + [ + "def foo():", + " return bar()", + "def bar():", + " raise RuntimeError('hello!')", + "foo()", + ] + ) + ) out, err = tt.ipexec(path) assert "execfile" not in out assert "RuntimeError" in out diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index 6e855d10186..e10abb863a1 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -58,7 +58,7 @@ def test_changing_py_file(self): """ with TemporaryDirectory() as td: fname = os.path.join(td, "foo.py") - with open(fname, "w", encoding='utf-8') as f: + with open(fname, "w", encoding="utf-8") as f: f.write(file_1) with prepended_to_syspath(td): @@ -68,7 +68,7 @@ def test_changing_py_file(self): ip.run_cell("foo.f()") # Make the file shorter, so the line of the error is missing. - with open(fname, "w", encoding='utf-8') as f: + with open(fname, "w", encoding="utf-8") as f: f.write(file_2) # For some reason, this was failing on the *second* call after @@ -92,7 +92,7 @@ def test_nonascii_path(self): # Non-ascii directory name as well. with TemporaryDirectory(suffix=u'é') as td: fname = os.path.join(td, u"fooé.py") - with open(fname, "w", encoding='utf-8') as f: + with open(fname, "w", encoding="utf-8") as f: f.write(file_1) with prepended_to_syspath(td): @@ -172,7 +172,7 @@ def test_indentationerror_shows_line(self): with TemporaryDirectory() as td: fname = os.path.join(td, "foo.py") - with open(fname, "w", encoding='utf-8') as f: + with open(fname, "w", encoding="utf-8") as f: f.write(indentationerror_file) with tt.AssertPrints("IndentationError"): @@ -221,14 +221,14 @@ def bar(): def test_changing_py_file(self): with TemporaryDirectory() as td: fname = os.path.join(td, "foo.py") - with open(fname, 'w', encoding='utf-8') as f: + with open(fname, "w", encoding="utf-8") as f: f.write(se_file_1) with tt.AssertPrints(["7/", "SyntaxError"]): ip.magic("run " + fname) # Modify the file - with open(fname, 'w', encoding='utf-8') as f: + with open(fname, "w", encoding="utf-8") as f: f.write(se_file_2) # The SyntaxError should point to the correct line diff --git a/IPython/extensions/storemagic.py b/IPython/extensions/storemagic.py index ff124ee38bd..d9d00f14b9a 100644 --- a/IPython/extensions/storemagic.py +++ b/IPython/extensions/storemagic.py @@ -176,12 +176,12 @@ def store(self, parameter_s=''): # default action - store the variable else: # %store foo >file.txt or >>file.txt - if len(args) > 1 and args[1].startswith('>'): - fnam = os.path.expanduser(args[1].lstrip('>').lstrip()) - if args[1].startswith('>>'): - fil = open(fnam, 'a', encoding='utf-8') + if len(args) > 1 and args[1].startswith(">"): + fnam = os.path.expanduser(args[1].lstrip(">").lstrip()) + if args[1].startswith(">>"): + fil = open(fnam, "a", encoding="utf-8") else: - fil = open(fnam, 'w', encoding='utf-8') + fil = open(fnam, "w", encoding="utf-8") with fil: obj = ip.ev(args[0]) print("Writing '%s' (%s) to file '%s'." % (args[0], diff --git a/IPython/extensions/tests/test_autoreload.py b/IPython/extensions/tests/test_autoreload.py index 63bae802524..a0fe725aa68 100644 --- a/IPython/extensions/tests/test_autoreload.py +++ b/IPython/extensions/tests/test_autoreload.py @@ -119,13 +119,13 @@ def write_file(self, filename, content): time.sleep(1.05) # Write - with open(filename, "w", encoding='utf-8') as f: + with open(filename, "w", encoding="utf-8") as f: f.write(content) def new_module(self, code): code = textwrap.dedent(code) mod_name, mod_fn = self.get_module() - with open(mod_fn, "w", encoding='utf-8') as f: + with open(mod_fn, "w", encoding="utf-8") as f: f.write(code) return mod_name, mod_fn diff --git a/IPython/lib/demo.py b/IPython/lib/demo.py index 0c96de6e72d..8c9ae905d49 100644 --- a/IPython/lib/demo.py +++ b/IPython/lib/demo.py @@ -405,7 +405,7 @@ def edit(self,index=None): filename = self.shell.mktempfile(self.src_blocks[index]) self.shell.hooks.editor(filename, 1) - with open(Path(filename), "r", encoding='utf-8') as f: + with open(Path(filename), "r", encoding="utf-8") as f: new_block = f.read() # update the source and colored block self.src_blocks[index] = new_block diff --git a/IPython/lib/tests/test_deepreload.py b/IPython/lib/tests/test_deepreload.py index f86e1027f98..827249cbad0 100644 --- a/IPython/lib/tests/test_deepreload.py +++ b/IPython/lib/tests/test_deepreload.py @@ -19,9 +19,9 @@ def test_deepreload(): with TemporaryDirectory() as tmpdir: with prepended_to_syspath(tmpdir): tmpdirpath = Path(tmpdir) - with open(tmpdirpath / "A.py", "w", encoding='utf-8') as f: + with open(tmpdirpath / "A.py", "w", encoding="utf-8") as f: f.write("class Object:\n pass\nok = True\n") - with open(tmpdirpath / "B.py", "w", encoding='utf-8') as f: + with open(tmpdirpath / "B.py", "w", encoding="utf-8") as f: f.write("import A\nassert A.ok, 'we are fine'\n") import A import B diff --git a/IPython/testing/tests/test_tools.py b/IPython/testing/tests/test_tools.py index 0e66ca39a99..178863cf686 100644 --- a/IPython/testing/tests/test_tools.py +++ b/IPython/testing/tests/test_tools.py @@ -62,7 +62,7 @@ def test_temp_pyfile(): src = 'pass\n' fname = tt.temp_pyfile(src) assert os.path.isfile(fname) - with open(fname, encoding='utf-8') as fh2: + with open(fname, encoding="utf-8") as fh2: src2 = fh2.read() assert src2 == src diff --git a/IPython/testing/tools.py b/IPython/testing/tools.py index fafb73fb8f7..2ff63a6d4a0 100644 --- a/IPython/testing/tools.py +++ b/IPython/testing/tools.py @@ -426,9 +426,8 @@ def mute_warn(): @contextmanager def make_tempfile(name): - """ Create an empty, named, temporary file for the duration of the context. - """ - open(name, 'w', encoding='utf-8').close() + """Create an empty, named, temporary file for the duration of the context.""" + open(name, "w", encoding="utf-8").close() try: yield finally: diff --git a/IPython/tests/cve.py b/IPython/tests/cve.py index aba6e20fa79..0a9dec4e854 100644 --- a/IPython/tests/cve.py +++ b/IPython/tests/cve.py @@ -33,7 +33,9 @@ def test_cve_2022_21699(): with TemporaryWorkingDirectory() as t: dangerous_startup_dir.mkdir(parents=True) - (dangerous_startup_dir / "foo.py").write_text(f'print("{dangerous_expected}")', encoding='utf-8') + (dangerous_startup_dir / "foo.py").write_text( + f'print("{dangerous_expected}")', encoding="utf-8" + ) # 1 sec to make sure FS is flushed. # time.sleep(1) cmd = [sys.executable, "-m", "IPython"] diff --git a/IPython/utils/io.py b/IPython/utils/io.py index ef1be80988f..170bc625acb 100644 --- a/IPython/utils/io.py +++ b/IPython/utils/io.py @@ -20,7 +20,7 @@ from .capture import CapturedIO, capture_output # setup stdin/stdout/stderr to sys.stdin/sys.stdout/sys.stderr -devnull = open(os.devnull, 'w', encoding='utf-8') +devnull = open(os.devnull, "w", encoding="utf-8") atexit.register(devnull.close) @@ -52,7 +52,7 @@ def __init__(self, file_or_name, mode="w", channel='stdout'): if hasattr(file_or_name, 'write') and hasattr(file_or_name, 'seek'): self.file = file_or_name else: - encoding = None if 'b' in mode else 'utf-8' + encoding = None if "b" in mode else "utf-8" self.file = open(file_or_name, mode, encoding=encoding) self.channel = channel self.ostream = getattr(sys, channel) @@ -132,7 +132,7 @@ def temp_pyfile(src, ext='.py'): It is the caller's responsibility to close the open file and unlink it. """ fname = tempfile.mkstemp(ext)[1] - with open(Path(fname), "w", encoding='utf-8') as f: + with open(Path(fname), "w", encoding="utf-8") as f: f.write(src) f.flush() return fname diff --git a/IPython/utils/tempdir.py b/IPython/utils/tempdir.py index 0729a8417dc..c3918d05bb0 100644 --- a/IPython/utils/tempdir.py +++ b/IPython/utils/tempdir.py @@ -24,7 +24,7 @@ def __init__(self, filename, mode='w+b', bufsize=-1, **kwds): """ self._tmpdir = TemporaryDirectory(**kwds) path = Path(self._tmpdir.name) / filename - encoding = None if 'b' in mode else 'utf-8' + encoding = None if "b" in mode else "utf-8" self.file = open(path, mode, bufsize, encoding=encoding) def cleanup(self): diff --git a/IPython/utils/tests/test_module_paths.py b/IPython/utils/tests/test_module_paths.py index 2abf889514c..8438a1e737f 100644 --- a/IPython/utils/tests/test_module_paths.py +++ b/IPython/utils/tests/test_module_paths.py @@ -32,7 +32,7 @@ old_syspath = sys.path def make_empty_file(fname): - open(fname, 'w', encoding='utf-8').close() + open(fname, "w", encoding="utf-8").close() def setup_module(): diff --git a/IPython/utils/tests/test_path.py b/IPython/utils/tests/test_path.py index f92e6ae1f6c..13e322320e0 100644 --- a/IPython/utils/tests/test_path.py +++ b/IPython/utils/tests/test_path.py @@ -295,7 +295,7 @@ def test_not_writable_ipdir(self): ipdir = os.path.join(tmpdir, '.ipython') os.mkdir(ipdir, 0o555) try: - open(os.path.join(ipdir, "_foo_"), 'w', encoding='utf-8').close() + open(os.path.join(ipdir, "_foo_"), "w", encoding="utf-8").close() except IOError: pass else: @@ -352,7 +352,7 @@ def setUpClass(cls): with cls.in_tempdir(): # Create empty files for fname in cls.filenames: - open(os.path.join(td, fname), 'w', encoding='utf-8').close() + open(os.path.join(td, fname), "w", encoding="utf-8").close() @classmethod def tearDownClass(cls): @@ -426,9 +426,9 @@ def test_ensure_dir_exists(): d = os.path.join(td, '∂ir') path.ensure_dir_exists(d) # create it assert os.path.isdir(d) - path.ensure_dir_exists(d) # no-op - f = os.path.join(td, 'ƒile') - open(f, 'w', encoding='utf-8').close() # touch + path.ensure_dir_exists(d) # no-op + f = os.path.join(td, "ƒile") + open(f, "w", encoding="utf-8").close() # touch with pytest.raises(IOError): path.ensure_dir_exists(f) @@ -436,7 +436,7 @@ class TestLinkOrCopy(unittest.TestCase): def setUp(self): self.tempdir = TemporaryDirectory() self.src = self.dst("src") - with open(self.src, "w", encoding='utf-8') as f: + with open(self.src, "w", encoding="utf-8") as f: f.write("Hello, world!") def tearDown(self): @@ -456,8 +456,8 @@ def assert_inode_equal(self, a, b): ), "%r and %r do not reference the same indoes" % (a, b) def assert_content_equal(self, a, b): - with open(a, 'rb') as a_f: - with open(b, 'rb') as b_f: + with open(a, "rb") as a_f: + with open(b, "rb") as b_f: assert a_f.read() == b_f.read() @skip_win32 @@ -477,7 +477,7 @@ def test_link_into_dir(self): @skip_win32 def test_target_exists(self): dst = self.dst("target") - open(dst, "w", encoding='utf-8').close() + open(dst, "w", encoding="utf-8").close() path.link_or_copy(self.src, dst) self.assert_inode_equal(self.src, dst) diff --git a/docs/autogen_config.py b/docs/autogen_config.py index 56733f3b422..43c38dd2d43 100755 --- a/docs/autogen_config.py +++ b/docs/autogen_config.py @@ -102,7 +102,7 @@ def reverse_aliases(app): def write_doc(name, title, app, preamble=None): trait_aliases = reverse_aliases(app) filename = options / (name + ".rst") - with open(filename, "w", encoding='utf-8') as f: + with open(filename, "w", encoding="utf-8") as f: f.write(title + "\n") f.write(("=" * len(title)) + "\n") f.write("\n") @@ -117,7 +117,7 @@ def write_doc(name, title, app, preamble=None): if __name__ == '__main__': # Touch this file for the make target - Path(generated).write_text("", encoding='utf-8') + Path(generated).write_text("", encoding="utf-8") write_doc('terminal', 'Terminal IPython options', TerminalIPythonApp()) write_doc('kernel', 'IPython kernel options', IPKernelApp(), diff --git a/docs/autogen_magics.py b/docs/autogen_magics.py index bfeb6aa7733..6102d0950c6 100644 --- a/docs/autogen_magics.py +++ b/docs/autogen_magics.py @@ -63,4 +63,4 @@ def sortkey(s): return s[0].lower() src_path = Path(__file__).parent dest = src_path.joinpath("source", "interactive", "magics-generated.txt") -dest.write_text("\n".join(output), encoding='utf-8') +dest.write_text("\n".join(output), encoding="utf-8") diff --git a/docs/autogen_shortcuts.py b/docs/autogen_shortcuts.py index 7ee0f910768..db7fe8d4917 100755 --- a/docs/autogen_shortcuts.py +++ b/docs/autogen_shortcuts.py @@ -89,6 +89,8 @@ def sort_key(item): (single_filter, "single_filtered"), (multi_filter, "multi_filtered"), ]: - with (dest / "{}.csv".format(output_filename)).open("w", encoding='utf-8') as csv: + with (dest / "{}.csv".format(output_filename)).open( + "w", encoding="utf-8" + ) as csv: for (shortcut, flt), v in sorted(filters.items(), key=sort_key): csv.write(":kbd:`{}`\t{}\t{}\n".format(shortcut, flt, v)) diff --git a/docs/source/conf.py b/docs/source/conf.py index ba9aa1dbc88..29212af8bf7 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,11 +28,14 @@ for name in ("config", "api", "magics", "shortcuts"): fname = Path("autogen_{}.py".format(name)) fpath = (Path(__file__).parent).joinpath("..", fname) - with open(fpath, encoding='utf-8') as f: - exec(compile(f.read(), fname, 'exec'), { - '__file__': fpath, - '__name__': '__main__', - }) + with open(fpath, encoding="utf-8") as f: + exec( + compile(f.read(), fname, "exec"), + { + "__file__": fpath, + "__name__": "__main__", + }, + ) else: import sphinx_rtd_theme html_theme = "sphinx_rtd_theme" @@ -45,7 +48,14 @@ # We load the ipython release info into a dict by explicit execution iprelease = {} -exec(compile(open('../../IPython/core/release.py', encoding='utf-8').read(), '../../IPython/core/release.py', 'exec'),iprelease) +exec( + compile( + open("../../IPython/core/release.py", encoding="utf-8").read(), + "../../IPython/core/release.py", + "exec", + ), + iprelease, +) # General configuration # --------------------- diff --git a/docs/sphinxext/apigen.py b/docs/sphinxext/apigen.py index 71a384b7d2a..e58493b17fd 100644 --- a/docs/sphinxext/apigen.py +++ b/docs/sphinxext/apigen.py @@ -390,9 +390,8 @@ def write_modules_api(self, modules,outdir): if not api_str: continue # write out to file - outfile = os.path.join(outdir, - m + self.rst_extension) - with open(outfile, 'wt', encoding='utf-8') as fileobj: + outfile = os.path.join(outdir, m + self.rst_extension) + with open(outfile, "wt", encoding="utf-8") as fileobj: fileobj.write(api_str) written_modules.append(m) self.written_modules = written_modules @@ -444,7 +443,7 @@ def write_index(self, outdir, path='gen.rst', relative_to=None): relpath = outdir.replace(relative_to + os.path.sep, '') else: relpath = outdir - with open(path,'wt', encoding='utf-8') as idx: + with open(path, "wt", encoding="utf-8") as idx: w = idx.write w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n') w('.. autosummary::\n' diff --git a/examples/IPython Kernel/ipython-get-history.py b/examples/IPython Kernel/ipython-get-history.py index fe38081fbad..116b4b7c155 100755 --- a/examples/IPython Kernel/ipython-get-history.py +++ b/examples/IPython Kernel/ipython-get-history.py @@ -24,7 +24,7 @@ session_number = int(sys.argv[1]) if len(sys.argv) > 2: filepath = Path(sys.argv[2]) - dest = open(filepath, "w", encoding='utf-8') + dest = open(filepath, "w", encoding="utf-8") raw = not filepath.name.endswith(".py") else: dest = sys.stdout diff --git a/setupbase.py b/setupbase.py index 788ce2491fb..b57dcc1b2a5 100644 --- a/setupbase.py +++ b/setupbase.py @@ -36,7 +36,7 @@ def execfile(fname, globs, locs=None): locs = locs or globs - with open(fname, encoding='utf-8') as f: + with open(fname, encoding="utf-8") as f: exec(compile(f.read(), fname, "exec"), globs, locs) # A little utility we'll need below, since glob() does NOT allow you to do @@ -336,10 +336,13 @@ def _record_commit(self, base_dir): os.remove(out_pth) except (IOError, OSError): pass - with open(out_pth, 'w', encoding='utf-8') as out_file: - out_file.writelines([ - '# GENERATED BY setup.py\n', - 'commit = u"%s"\n' % repo_commit, - ]) + with open(out_pth, "w", encoding="utf-8") as out_file: + out_file.writelines( + [ + "# GENERATED BY setup.py\n", + 'commit = u"%s"\n' % repo_commit, + ] + ) + return MyBuildPy diff --git a/tools/fixup_whats_new_pr.py b/tools/fixup_whats_new_pr.py index 397c0bf38e3..9b74da8055c 100644 --- a/tools/fixup_whats_new_pr.py +++ b/tools/fixup_whats_new_pr.py @@ -22,14 +22,14 @@ def main(): print("Adding pseudo-title to:", filepath.name) title = filepath.name[:-4].split("/")[-1].replace("-", " ").capitalize() - data = filepath.read_text(encoding='utf-8') + data = filepath.read_text(encoding="utf-8") try: if data and data.splitlines()[1].startswith('='): continue except IndexError: pass - with filepath.open("w", encoding='utf-8') as f: + with filepath.open("w", encoding="utf-8") as f: f.write(title + "\n") f.write("=" * len(title) + "\n\n") f.write(data) diff --git a/tools/toollib.py b/tools/toollib.py index c77bbbc6853..f32e06a5fa5 100644 --- a/tools/toollib.py +++ b/tools/toollib.py @@ -45,4 +45,4 @@ def get_ipdir(): def execfile(fname, globs, locs=None): locs = locs or globs - exec(compile(open(fname, encoding='utf-8').read(), fname, "exec"), globs, locs) + exec(compile(open(fname, encoding="utf-8").read(), fname, "exec"), globs, locs) From 7a171250be20214526089e7638a6002e83a3fcf4 Mon Sep 17 00:00:00 2001 From: gousaiyang Date: Fri, 4 Feb 2022 22:53:36 -0800 Subject: [PATCH 51/85] IPython currently incompatible with pytest 7 --- .github/workflows/downstream.yml | 2 +- docs/requirements.txt | 2 +- setup.cfg | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 168b32e2678..5c5adc1e0ab 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -41,7 +41,7 @@ jobs: python -m pip install --upgrade -e file://$PWD#egg=ipython[test] # we must instal IPython after ipykernel to get the right versions. python -m pip install --upgrade --upgrade-strategy eager flaky ipyparallel - python -m pip install --upgrade pytest + python -m pip install --upgrade 'pytest<7' - name: pytest env: COLUMNS: 120 diff --git a/docs/requirements.txt b/docs/requirements.txt index 93e162f6660..587288c2a0f 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ sphinx-rtd-theme docrepr matplotlib stack_data -pytest +pytest<7 diff --git a/setup.cfg b/setup.cfg index 2da02e4e598..4fbffaa1a07 100644 --- a/setup.cfg +++ b/setup.cfg @@ -63,7 +63,7 @@ qtconsole = qtconsole terminal = test = - pytest + pytest<7 pytest-asyncio testpath test_extra = @@ -72,7 +72,7 @@ test_extra = nbformat numpy>=1.19 pandas - pytest + pytest<7 testpath trio all = From 65f7fdfefa2c9ff872bcbd63f827b9256ed28594 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 7 Feb 2022 11:26:44 +0100 Subject: [PATCH 52/85] pin pytest --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 2da02e4e598..4fbffaa1a07 100644 --- a/setup.cfg +++ b/setup.cfg @@ -63,7 +63,7 @@ qtconsole = qtconsole terminal = test = - pytest + pytest<7 pytest-asyncio testpath test_extra = @@ -72,7 +72,7 @@ test_extra = nbformat numpy>=1.19 pandas - pytest + pytest<7 testpath trio all = From e306c9d3f707de42b47a1e7c4c8034d6862fba5f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 7 Feb 2022 11:21:10 +0100 Subject: [PATCH 53/85] Update old deprecation --- IPython/core/interactiveshell.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index ecb9c5f1652..902da1bd9a1 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2287,7 +2287,11 @@ def find_magic(self, magic_name, magic_kind='line'): return self.magics_manager.magics[magic_kind].get(magic_name) def magic(self, arg_s): - """DEPRECATED. Use run_line_magic() instead. + """ + DEPRECATED + + Deprecated since IPython 0.13 (warning added in + 8.1), use run_line_magic(magic_name, parameter_s). Call a magic function by name. @@ -2305,6 +2309,12 @@ def magic(self, arg_s): valid Python code you can type at the interpreter, including loops and compound statements. """ + warnings.warn( + "`magic(...)` is deprecated since IPython 0.13 (warning added in " + "8.1), use run_line_magic(magic_name, parameter_s).", + DeprecationWarning, + stacklevel=2, + ) # TODO: should we issue a loud deprecation warning here? magic_name, _, magic_arg_s = arg_s.partition(' ') magic_name = magic_name.lstrip(prefilter.ESC_MAGIC) From e65762abb111da59c215c1382b8a3a601c57eea6 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 8 Feb 2022 11:32:28 +0100 Subject: [PATCH 54/85] Make unicode backslach completion fuzzy and case insensitive Ok, the fuzzy part is super simple. - candidates matching prefix - If no matches: names that contain the string - If no matches :whether each part of the string split on space is contained in the name of the unicode character. That is to say, `\GREEK OMICRON` will search whether both GREEK and and OMICRON (case insensitive) are available, in the worst case scenario This allows things like `\omicron` to give you proper suggestions. `\nu` will give you latex nu, `\greek nu` with match as the prefix of `GREEK NUMERAL ...` `\Nu` will match all the `... NUMERAL...` in unicode, but `\Nu greek` will limit the searches enough Mitigate #13514 --- IPython/core/completer.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index ccb1f465f5b..9774fd5a4e7 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -2222,12 +2222,22 @@ def fwd_unicode_match(self, text:str) -> Tuple[str, Sequence[str]]: # initialized, and it takes a user-noticeable amount of time to # initialize it, so we don't want to initialize it unless we're # actually going to use it. - s = text[slashpos+1:] - candidates = [x for x in self.unicode_names if x.startswith(s)] + s = text[slashpos + 1 :] + sup = s.upper() + candidates = [x for x in self.unicode_names if x.startswith(sup)] if candidates: return s, candidates - else: - return '', () + candidates = [x for x in self.unicode_names if sup in x] + if candidates: + return s, candidates + splitsup = sup.split(" ") + candidates = [ + x for x in self.unicode_names if all(u in x for u in splitsup) + ] + if candidates: + return s, candidates + + return "", () # if text does not start with slash else: From e5bbea060db04b6a3d578e6a444278c981dc1db5 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 1 Feb 2022 15:14:34 +0100 Subject: [PATCH 55/85] Allow to configure lazy loadable magics. While we encourage load_ipython_ext to be lazy and not take too much resources, it might not be practical especially when the top level module takes a long time to import. Here we allow to define a mapping between magics names and extension name, and on attempt to execute a non-existing magics we'll look into the lazy mapping and try to load it. We also add a helper to let us interactively register a magic lazily, see the `register_lazy()` method of to magics_manager --- IPython/core/interactiveshell.py | 41 +++++++++++++++++--- IPython/core/magic.py | 46 +++++++++++++++++++++++ IPython/core/tests/test_magic.py | 44 +++++++++++++++++++++- IPython/core/tests/test_magic_terminal.py | 35 ++++++++++++++--- IPython/terminal/ipapp.py | 2 + docs/source/whatsnew/version7.rst | 18 +++++++++ 6 files changed, 175 insertions(+), 11 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 902da1bd9a1..3d9ec35b2ed 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -500,7 +500,6 @@ def profile(self): def __init__(self, ipython_dir=None, profile_dir=None, user_module=None, user_ns=None, custom_exceptions=((), None), **kwargs): - # This is where traits with a config_key argument are updated # from the values on config. super(InteractiveShell, self).__init__(**kwargs) @@ -1643,7 +1642,7 @@ def _inspect(self, meth, oname, namespaces=None, **kw): formatter, info, enable_html_pager=self.enable_html_pager, - **kw + **kw, ) else: pmethod(info.obj, oname) @@ -2173,7 +2172,34 @@ def register_magic_function(self, func, magic_kind='line', magic_name=None): func, magic_kind=magic_kind, magic_name=magic_name ) - def run_line_magic(self, magic_name, line, _stack_depth=1): + def _find_with_lazy_load(self, /, type_, magic_name: str): + """ + Try to find a magic potentially lazy-loading it. + + Parameters + ---------- + + type_: "line"|"cell" + the type of magics we are trying to find/lazy load. + magic_name: str + The name of the magic we are trying to find/lazy load + + + Note that this may have any side effects + """ + finder = {"line": self.find_line_magic, "cell": self.find_cell_magic}[type_] + fn = finder(magic_name) + if fn is not None: + return fn + lazy = self.magics_manager.lazy_magics.get(magic_name) + if lazy is None: + return None + + self.run_line_magic("load_ext", lazy) + res = finder(magic_name) + return res + + def run_line_magic(self, magic_name: str, line, _stack_depth=1): """Execute the given line magic. Parameters @@ -2186,7 +2212,12 @@ def run_line_magic(self, magic_name, line, _stack_depth=1): If run_line_magic() is called from magic() then _stack_depth=2. This is added to ensure backward compatibility for use of 'get_ipython().magic()' """ - fn = self.find_line_magic(magic_name) + fn = self._find_with_lazy_load("line", magic_name) + if fn is None: + lazy = self.magics_manager.lazy_magics.get(magic_name) + if lazy: + self.run_line_magic("load_ext", lazy) + fn = self.find_line_magic(magic_name) if fn is None: cm = self.find_cell_magic(magic_name) etpl = "Line magic function `%%%s` not found%s." @@ -2237,7 +2268,7 @@ def run_cell_magic(self, magic_name, line, cell): cell : str The body of the cell as a (possibly multiline) string. """ - fn = self.find_cell_magic(magic_name) + fn = self._find_with_lazy_load("cell", magic_name) if fn is None: lm = self.find_line_magic(magic_name) etpl = "Cell magic `%%{0}` not found{1}." diff --git a/IPython/core/magic.py b/IPython/core/magic.py index 3dc3480b561..79983df09f2 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -302,6 +302,34 @@ class MagicsManager(Configurable): # holding the actual callable object as value. This is the dict used for # magic function dispatch magics = Dict() + lazy_magics = Dict( + help=""" + Mapping from magic names to modules to load. + + This can be used in IPython/IPykernel configuration to declare lazy magics + that will only be imported/registered on first use. + + For example:: + + c.MagicsManger.lazy_magics = { + "my_magic": "slow.to.import", + "my_other_magic": "also.slow", + } + + On first invocation of `%my_magic`, `%%my_magic`, `%%my_other_magic` or + `%%my_other_magic`, the corresponding module will be loaded as an ipython + extensions as if you had previously done `%load_ext ipython`. + + Magics names should be without percent(s) as magics can be both cell + and line magics. + + Lazy loading happen relatively late in execution process, and + complex extensions that manipulate Python/IPython internal state or global state + might not support lazy loading. + """ + ).tag( + config=True, + ) # A registry of the original objects that we've been given holding magics. registry = Dict() @@ -366,6 +394,24 @@ def lsmagic_docs(self, brief=False, missing=''): docs[m_type] = m_docs return docs + def register_lazy(self, name: str, fully_qualified_name: str): + """ + Lazily register a magic via an extension. + + + Parameters + ---------- + name : str + Name of the magic you wish to register. + fully_qualified_name : + Fully qualified name of the module/submodule that should be loaded + as an extensions when the magic is first called. + It is assumed that loading this extensions will register the given + magic. + """ + + self.lazy_magics[name] = fully_qualified_name + def register(self, *magic_objects): """Register one or more instances of Magics. diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index 5294f82c51c..cbcb5c1411c 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -34,9 +34,11 @@ from IPython.utils.io import capture_output from IPython.utils.process import find_cmd from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory +from IPython.utils.syspathcontext import prepended_to_syspath from .test_debugger import PdbTestInput +from tempfile import NamedTemporaryFile @magic.magics_class class DummyMagics(magic.Magics): pass @@ -1325,13 +1327,53 @@ def test_timeit_arguments(): _ip.magic("timeit -n1 -r1 a=('#')") +MINIMAL_LAZY_MAGIC = """ +from IPython.core.magic import ( + Magics, + magics_class, + line_magic, + cell_magic, +) + + +@magics_class +class LazyMagics(Magics): + @line_magic + def lazy_line(self, line): + print("Lazy Line") + + @cell_magic + def lazy_cell(self, line, cell): + print("Lazy Cell") + + +def load_ipython_extension(ipython): + ipython.register_magics(LazyMagics) +""" + + +def test_lazy_magics(): + with pytest.raises(UsageError): + ip.run_line_magic("lazy_line", "") + + startdir = os.getcwd() + + with TemporaryDirectory() as tmpdir: + with prepended_to_syspath(tmpdir): + ptempdir = Path(tmpdir) + tf = ptempdir / "lazy_magic_module.py" + tf.write_text(MINIMAL_LAZY_MAGIC) + ip.magics_manager.register_lazy("lazy_line", Path(tf.name).name[:-3]) + with tt.AssertPrints("Lazy Line"): + ip.run_line_magic("lazy_line", "") + + TEST_MODULE = """ print('Loaded my_tmp') if __name__ == "__main__": print('I just ran a script') """ - def test_run_module_from_import_hook(): "Test that a module can be loaded via an import hook" with TemporaryDirectory() as tmpdir: diff --git a/IPython/core/tests/test_magic_terminal.py b/IPython/core/tests/test_magic_terminal.py index 721fd5eda4d..f09014786e6 100644 --- a/IPython/core/tests/test_magic_terminal.py +++ b/IPython/core/tests/test_magic_terminal.py @@ -9,11 +9,35 @@ from unittest import TestCase from IPython.testing import tools as tt - #----------------------------------------------------------------------------- # Test functions begin #----------------------------------------------------------------------------- + +MINIMAL_LAZY_MAGIC = """ +from IPython.core.magic import ( + Magics, + magics_class, + line_magic, + cell_magic, +) + + +@magics_class +class LazyMagics(Magics): + @line_magic + def lazy_line(self, line): + print("Lazy Line") + + @cell_magic + def lazy_cell(self, line, cell): + print("Lazy Cell") + + +def load_ipython_extension(ipython): + ipython.register_magics(LazyMagics) +""" + def check_cpaste(code, should_fail=False): """Execute code via 'cpaste' and ensure it was executed, unless should_fail is set. @@ -31,7 +55,7 @@ def check_cpaste(code, should_fail=False): try: context = tt.AssertPrints if should_fail else tt.AssertNotPrints with context("Traceback (most recent call last)"): - ip.magic('cpaste') + ip.run_line_magic("cpaste", "") if not should_fail: assert ip.user_ns['code_ran'], "%r failed" % code @@ -68,13 +92,14 @@ def runf(): check_cpaste(code, should_fail=True) + class PasteTestCase(TestCase): """Multiple tests for clipboard pasting""" def paste(self, txt, flags='-q'): """Paste input text, by default in quiet mode""" - ip.hooks.clipboard_get = lambda : txt - ip.magic('paste '+flags) + ip.hooks.clipboard_get = lambda: txt + ip.run_line_magic("paste", flags) def setUp(self): # Inject fake clipboard hook but save original so we can restore it later @@ -114,7 +139,7 @@ def test_paste_py_multi_r(self): self.assertEqual(ip.user_ns.pop("x"), [1, 2, 3]) self.assertEqual(ip.user_ns.pop("y"), [1, 4, 9]) self.assertFalse("x" in ip.user_ns) - ip.magic("paste -r") + ip.run_line_magic("paste", "-r") self.assertEqual(ip.user_ns["x"], [1, 2, 3]) self.assertEqual(ip.user_ns["y"], [1, 4, 9]) diff --git a/IPython/terminal/ipapp.py b/IPython/terminal/ipapp.py index ed39b7dc5d0..e735a209255 100755 --- a/IPython/terminal/ipapp.py +++ b/IPython/terminal/ipapp.py @@ -25,6 +25,7 @@ from IPython.core.application import ( ProfileDir, BaseIPythonApplication, base_flags, base_aliases ) +from IPython.core.magic import MagicsManager from IPython.core.magics import ( ScriptMagics, LoggingMagics ) @@ -200,6 +201,7 @@ def _classes_default(self): self.__class__, # it will also affect subclasses (e.g. QtConsole) TerminalInteractiveShell, HistoryManager, + MagicsManager, ProfileDir, PlainTextFormatter, IPCompleter, diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 55b1b2f8439..6597fe555f8 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -3,6 +3,24 @@ ============ +.. _version 7.32: + +IPython 7.32 +============ + + +The ability to configure magics to be lazily loaded has been added to IPython. +See the ``ipython --help-all`` section on ``MagicsManager.lazy_magic``. +One can now use:: + + c.MagicsManger.lazy_magics = { + "my_magic": "slow.to.import", + "my_other_magic": "also.slow", + } + +And on first use of ``%my_magic``, or corresponding cell magic, or other line magic, +the corresponding ``load_ext`` will be called just before trying to invoke the magic. + .. _version 7.31: IPython 7.31 From a96dd9e4724f4c425a14257e0f592a1f79a1ff0e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 8 Feb 2022 12:10:55 +0100 Subject: [PATCH 56/85] MAINT: deprecate append_to_syspath --- IPython/utils/syspathcontext.py | 11 ++++++++++- IPython/utils/tests/test_deprecated.py | 7 +++++++ IPython/utils/version.py | 5 ++++- 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 IPython/utils/tests/test_deprecated.py diff --git a/IPython/utils/syspathcontext.py b/IPython/utils/syspathcontext.py index bd1c51500d6..7af1ab60af9 100644 --- a/IPython/utils/syspathcontext.py +++ b/IPython/utils/syspathcontext.py @@ -15,12 +15,21 @@ #----------------------------------------------------------------------------- import sys +import warnings class appended_to_syspath(object): - """A context for appending a directory to sys.path for a second.""" + """ + Deprecated since IPython 8.1, no replacements. + + A context for appending a directory to sys.path for a second.""" def __init__(self, dir): + warnings.warn( + "`appended_to_syspath` is deprecated since IPython 8.1, and has no replacements", + DeprecationWarning, + stacklevel=2, + ) self.dir = dir def __enter__(self): diff --git a/IPython/utils/tests/test_deprecated.py b/IPython/utils/tests/test_deprecated.py new file mode 100644 index 00000000000..f6f54ce52a9 --- /dev/null +++ b/IPython/utils/tests/test_deprecated.py @@ -0,0 +1,7 @@ +from IPython.utils.syspathcontext import appended_to_syspath +import pytest + + +def test_append_deprecated(): + with pytest.warns(DeprecationWarning): + appended_to_syspath(".") diff --git a/IPython/utils/version.py b/IPython/utils/version.py index 050155640d4..8c65c78e15e 100644 --- a/IPython/utils/version.py +++ b/IPython/utils/version.py @@ -14,7 +14,10 @@ from warnings import warn -warn("The `IPython.utils.version` module has been deprecated since IPython 8.0.") +warn( + "The `IPython.utils.version` module has been deprecated since IPython 8.0.", + DeprecationWarning, +) def check_version(v, check): From fcd299c5726988dfae1aa81f5e47348eb7300231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samuel=20Leli=C3=A8vre?= Date: Sun, 6 Feb 2022 15:23:06 +0100 Subject: [PATCH 57/85] Sync `latex_symbols.jl` url in two places Follow-up to #11399 and #11400. Make sure the comment at the top of https://github.com/ipython/ipython/blob/master/IPython/core/latex_symbols.py points to the file actually used in - https://github.com/ipython/ipython/edit/master/tools/gen_latex_symbols.py --- tools/gen_latex_symbols.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/gen_latex_symbols.py b/tools/gen_latex_symbols.py index e1b441054ac..b623755136a 100644 --- a/tools/gen_latex_symbols.py +++ b/tools/gen_latex_symbols.py @@ -58,18 +58,18 @@ def test_ident(i): # Write the `latex_symbols.py` module in the cwd -s = """# encoding: utf-8 +s = f"""# encoding: utf-8 # DO NOT EDIT THIS FILE BY HAND. # To update this file, run the script /tools/gen_latex_symbols.py using Python 3 # This file is autogenerated from the file: -# https://raw.githubusercontent.com/JuliaLang/julia/master/base/latex_symbols.jl +# {url} # This original list is filtered to remove any unicode characters that are not valid # Python identifiers. -latex_symbols = {\n +latex_symbols = {{\n """ for line in valid_idents: s += ' "%s" : "%s",\n' % (line[0], line[1]) From 16451266fc75855a617f61a4e7bf6d29330965b1 Mon Sep 17 00:00:00 2001 From: ltrujello Date: Sun, 30 Jan 2022 11:09:36 -0800 Subject: [PATCH 58/85] Fixes #13472 by restoring user's terminal cursor --- IPython/core/interactiveshell.py | 4 ++++ IPython/terminal/shortcuts.py | 7 +------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 3d9ec35b2ed..17aacfc3013 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -3679,6 +3679,10 @@ def atexit_operations(self): pass del self.tempdirs + # Restore user's cursor + if hasattr(self, "editing_mode") and self.editing_mode == 'vi': + sys.stdout.write("\x1b[0 q") + sys.stdout.flush() def cleanup(self): self.restore_sys_module_state() diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index ea204f8ddd8..2cceb0786be 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -336,12 +336,7 @@ def set_input_mode(self, mode): shape = {InputMode.NAVIGATION: 2, InputMode.REPLACE: 4}.get(mode, 6) cursor = "\x1b[{} q".format(shape) - if hasattr(sys.stdout, "_cli"): - write = sys.stdout._cli.output.write_raw - else: - write = sys.stdout.write - - write(cursor) + sys.stdout.write(cursor) sys.stdout.flush() self._input_mode = mode From 42e22f8e67a24dc274652afaab046c6d57654714 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 8 Feb 2022 12:25:55 +0100 Subject: [PATCH 59/85] MAINT: cleanup imports of tempdir. Many files were importing tempdir from IPython.utils, though all the changes are now upstream in Python. Import directly from tempdir. Use isort/darker on touched files to cleanup imports at the same time --- IPython/core/interactiveshell.py | 81 +++++++++++++++---------- IPython/core/tests/test_application.py | 2 +- IPython/core/tests/test_completerlib.py | 3 +- IPython/core/tests/test_extension.py | 3 +- IPython/core/tests/test_history.py | 8 ++- IPython/core/tests/test_logger.py | 3 +- IPython/core/tests/test_paths.py | 4 +- IPython/core/tests/test_profile.py | 7 +-- IPython/core/tests/test_run.py | 8 ++- IPython/core/tests/test_ultratb.py | 9 ++- IPython/lib/tests/test_deepreload.py | 9 +-- IPython/utils/tempdir.py | 3 +- IPython/utils/tests/test_path.py | 10 +-- 13 files changed, 85 insertions(+), 65 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 902da1bd9a1..7d5a7d340b9 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -20,34 +20,53 @@ import os import re import runpy +import subprocess import sys import tempfile import traceback import types -import subprocess import warnings +from ast import stmt from io import open as io_open - +from logging import error from pathlib import Path -from pickleshare import PickleShareDB +from typing import Callable +from typing import List as ListType +from typing import Optional, Tuple +from warnings import warn +from pickleshare import PickleShareDB +from tempfile import TemporaryDirectory +from traitlets import ( + Any, + Bool, + CaselessStrEnum, + Dict, + Enum, + Instance, + Integer, + List, + Type, + Unicode, + default, + observe, + validate, +) from traitlets.config.configurable import SingletonConfigurable from traitlets.utils.importstring import import_item -from IPython.core import oinspect -from IPython.core import magic -from IPython.core import page -from IPython.core import prefilter -from IPython.core import ultratb + +import IPython.core.hooks +from IPython.core import magic, oinspect, page, prefilter, ultratb from IPython.core.alias import Alias, AliasManager from IPython.core.autocall import ExitAutocall from IPython.core.builtin_trap import BuiltinTrap -from IPython.core.events import EventManager, available_events from IPython.core.compilerop import CachingCompiler, check_linecache_ipython from IPython.core.debugger import InterruptiblePdb from IPython.core.display_trap import DisplayTrap from IPython.core.displayhook import DisplayHook from IPython.core.displaypub import DisplayPublisher from IPython.core.error import InputRejected, UsageError +from IPython.core.events import EventManager, available_events from IPython.core.extensions import ExtensionManager from IPython.core.formatters import DisplayFormatter from IPython.core.history import HistoryManager @@ -59,31 +78,17 @@ from IPython.core.profiledir import ProfileDir from IPython.core.usage import default_banner from IPython.display import display +from IPython.paths import get_ipython_dir from IPython.testing.skipdoctest import skip_doctest -from IPython.utils import PyColorize -from IPython.utils import io -from IPython.utils import py3compat -from IPython.utils import openpy +from IPython.utils import PyColorize, io, openpy, py3compat from IPython.utils.decorators import undoc from IPython.utils.io import ask_yes_no from IPython.utils.ipstruct import Struct -from IPython.paths import get_ipython_dir -from IPython.utils.path import get_home_dir, get_py_filename, ensure_dir_exists -from IPython.utils.process import system, getoutput +from IPython.utils.path import ensure_dir_exists, get_home_dir, get_py_filename +from IPython.utils.process import getoutput, system from IPython.utils.strdispatch import StrDispatch from IPython.utils.syspathcontext import prepended_to_syspath -from IPython.utils.text import format_screen, LSString, SList, DollarFormatter -from IPython.utils.tempdir import TemporaryDirectory -from traitlets import ( - Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type, - observe, default, validate, Any -) -from warnings import warn -from logging import error -import IPython.core.hooks - -from typing import List as ListType, Tuple, Optional, Callable -from ast import stmt +from IPython.utils.text import DollarFormatter, LSString, SList, format_screen sphinxify: Optional[Callable] @@ -122,8 +127,13 @@ class ProvisionalWarning(DeprecationWarning): # we still need to run things using the asyncio eventloop, but there is no # async integration -from .async_helpers import _asyncio_runner, _pseudo_sync_runner -from .async_helpers import _curio_runner, _trio_runner, _should_be_async +from .async_helpers import ( + _asyncio_runner, + _curio_runner, + _pseudo_sync_runner, + _should_be_async, + _trio_runner, +) #----------------------------------------------------------------------------- # Globals @@ -2035,8 +2045,12 @@ def init_completer(self): (typically over the network by remote frontends). """ from IPython.core.completer import IPCompleter - from IPython.core.completerlib import (module_completer, - magic_run_completer, cd_completer, reset_completer) + from IPython.core.completerlib import ( + cd_completer, + magic_run_completer, + module_completer, + reset_completer, + ) self.Completer = IPCompleter(shell=self, namespace=self.user_ns, @@ -3344,8 +3358,9 @@ def enable_matplotlib(self, gui=None): make sense in all contexts, for example a terminal ipython can't display figures inline. """ - from IPython.core import pylabtools as pt from matplotlib_inline.backend_inline import configure_inline_support + + from IPython.core import pylabtools as pt gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select) if gui != 'inline': diff --git a/IPython/core/tests/test_application.py b/IPython/core/tests/test_application.py index 891908e98e7..07ad291659f 100644 --- a/IPython/core/tests/test_application.py +++ b/IPython/core/tests/test_application.py @@ -4,11 +4,11 @@ import os import tempfile +from tempfile import TemporaryDirectory from traitlets import Unicode from IPython.core.application import BaseIPythonApplication from IPython.testing import decorators as dec -from IPython.utils.tempdir import TemporaryDirectory @dec.onlyif_unicode_paths diff --git a/IPython/core/tests/test_completerlib.py b/IPython/core/tests/test_completerlib.py index b5508447895..827204ccfa6 100644 --- a/IPython/core/tests/test_completerlib.py +++ b/IPython/core/tests/test_completerlib.py @@ -14,8 +14,9 @@ import unittest from os.path import join +from tempfile import TemporaryDirectory + from IPython.core.completerlib import magic_run_completer, module_completion, try_import -from IPython.utils.tempdir import TemporaryDirectory from IPython.testing.decorators import onlyif_unicode_paths diff --git a/IPython/core/tests/test_extension.py b/IPython/core/tests/test_extension.py index 51db4b6258c..8297bc4d40c 100644 --- a/IPython/core/tests/test_extension.py +++ b/IPython/core/tests/test_extension.py @@ -1,8 +1,9 @@ import os.path +from tempfile import TemporaryDirectory + import IPython.testing.tools as tt from IPython.utils.syspathcontext import prepended_to_syspath -from IPython.utils.tempdir import TemporaryDirectory ext1_content = """ def load_ipython_extension(ip): diff --git a/IPython/core/tests/test_history.py b/IPython/core/tests/test_history.py index 6d6a1b1dd15..388ebc64acd 100644 --- a/IPython/core/tests/test_history.py +++ b/IPython/core/tests/test_history.py @@ -7,17 +7,19 @@ # stdlib import io -from pathlib import Path +import sqlite3 import sys import tempfile from datetime import datetime -import sqlite3 +from pathlib import Path +from tempfile import TemporaryDirectory # our own packages from traitlets.config.loader import Config -from IPython.utils.tempdir import TemporaryDirectory + from IPython.core.history import HistoryManager, extract_hist_ranges + def test_proper_default_encoding(): assert sys.getdefaultencoding() == "utf-8" diff --git a/IPython/core/tests/test_logger.py b/IPython/core/tests/test_logger.py index 71c4e6b60c0..10e462098be 100644 --- a/IPython/core/tests/test_logger.py +++ b/IPython/core/tests/test_logger.py @@ -2,9 +2,10 @@ """Test IPython.core.logger""" import os.path + import pytest +from tempfile import TemporaryDirectory -from IPython.utils.tempdir import TemporaryDirectory def test_logstart_inaccessible_file(): with pytest.raises(IOError): diff --git a/IPython/core/tests/test_paths.py b/IPython/core/tests/test_paths.py index d1366ee34ef..eb754b81529 100644 --- a/IPython/core/tests/test_paths.py +++ b/IPython/core/tests/test_paths.py @@ -6,11 +6,11 @@ import warnings from unittest.mock import patch -from testpath import modified_env, assert_isdir, assert_isfile +from tempfile import TemporaryDirectory +from testpath import assert_isdir, assert_isfile, modified_env from IPython import paths from IPython.testing.decorators import skip_win32 -from IPython.utils.tempdir import TemporaryDirectory TMP_TEST_DIR = os.path.realpath(tempfile.mkdtemp()) HOME_TEST_DIR = os.path.join(TMP_TEST_DIR, "home_test_dir") diff --git a/IPython/core/tests/test_profile.py b/IPython/core/tests/test_profile.py index 8dd58cc67f4..8734ec3513e 100644 --- a/IPython/core/tests/test_profile.py +++ b/IPython/core/tests/test_profile.py @@ -23,17 +23,16 @@ import shutil import sys import tempfile - from pathlib import Path from unittest import TestCase -from IPython.core.profileapp import list_profiles_in, list_bundled_profiles -from IPython.core.profiledir import ProfileDir +from tempfile import TemporaryDirectory +from IPython.core.profileapp import list_bundled_profiles, list_profiles_in +from IPython.core.profiledir import ProfileDir from IPython.testing import decorators as dec from IPython.testing import tools as tt from IPython.utils.process import getoutput -from IPython.utils.tempdir import TemporaryDirectory #----------------------------------------------------------------------------- # Globals diff --git a/IPython/core/tests/test_run.py b/IPython/core/tests/test_run.py index 0f73a781e3e..c7841759170 100644 --- a/IPython/core/tests/test_run.py +++ b/IPython/core/tests/test_run.py @@ -19,21 +19,22 @@ import functools import os import platform -from os.path import join as pjoin import random import string import sys import textwrap import unittest +from os.path import join as pjoin from unittest.mock import patch import pytest +from tempfile import TemporaryDirectory +from IPython.core import debugger from IPython.testing import decorators as dec from IPython.testing import tools as tt from IPython.utils.io import capture_output -from IPython.utils.tempdir import TemporaryDirectory -from IPython.core import debugger + def doctest_refbug(): """Very nasty problem with references held by multiple runs of a script. @@ -411,6 +412,7 @@ def test_run_nb_error(self): """Test %run notebook.ipynb error""" pytest.importorskip("nbformat") from nbformat import v4, writes + # %run when a file name isn't provided pytest.raises(Exception, _ip.magic, "run") diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index 423998019b5..4b31ef0cd93 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -3,21 +3,20 @@ """ import io import logging +import os.path import platform import re import sys -import os.path -from textwrap import dedent import traceback import unittest +from textwrap import dedent -from IPython.core.ultratb import ColorTB, VerboseTB - +from tempfile import TemporaryDirectory +from IPython.core.ultratb import ColorTB, VerboseTB from IPython.testing import tools as tt from IPython.testing.decorators import onlyif_unicode_paths from IPython.utils.syspathcontext import prepended_to_syspath -from IPython.utils.tempdir import TemporaryDirectory file_1 = """1 2 diff --git a/IPython/lib/tests/test_deepreload.py b/IPython/lib/tests/test_deepreload.py index 9759a7f69f7..b8097905787 100644 --- a/IPython/lib/tests/test_deepreload.py +++ b/IPython/lib/tests/test_deepreload.py @@ -4,14 +4,15 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. -import pytest import types - from pathlib import Path +import pytest +from tempfile import TemporaryDirectory + +from IPython.lib.deepreload import modules_reloading +from IPython.lib.deepreload import reload as dreload from IPython.utils.syspathcontext import prepended_to_syspath -from IPython.utils.tempdir import TemporaryDirectory -from IPython.lib.deepreload import reload as dreload, modules_reloading def test_deepreload(): diff --git a/IPython/utils/tempdir.py b/IPython/utils/tempdir.py index bbfffe9c923..b3e151de59e 100644 --- a/IPython/utils/tempdir.py +++ b/IPython/utils/tempdir.py @@ -10,8 +10,7 @@ class NamedFileInTemporaryDirectory(object): - - def __init__(self, filename, mode='w+b', bufsize=-1, **kwds): + def __init__(self, filename, mode="w+b", bufsize=-1, add_to_syspath=False, **kwds): """ Open a file named `filename` in a temporary directory. diff --git a/IPython/utils/tests/test_path.py b/IPython/utils/tests/test_path.py index b27e4355383..16fcd5f341f 100644 --- a/IPython/utils/tests/test_path.py +++ b/IPython/utils/tests/test_path.py @@ -10,24 +10,23 @@ import tempfile import unittest from contextlib import contextmanager -from unittest.mock import patch -from os.path import join, abspath from importlib import reload +from os.path import abspath, join +from unittest.mock import patch import pytest +from tempfile import TemporaryDirectory import IPython from IPython import paths from IPython.testing import decorators as dec from IPython.testing.decorators import ( + onlyif_unicode_paths, skip_if_not_win32, skip_win32, - onlyif_unicode_paths, ) from IPython.testing.tools import make_tempfile from IPython.utils import path -from IPython.utils.tempdir import TemporaryDirectory - # Platform-dependent imports try: @@ -41,6 +40,7 @@ import winreg as wreg except ImportError: import _winreg as wreg + #Add entries that needs to be stubbed by the testing code (wreg.OpenKey, wreg.QueryValueEx,) = (None, None) From 234b6375bf384c571eaacf984ed43503b98a8448 Mon Sep 17 00:00:00 2001 From: Nikita Kniazev Date: Wed, 9 Feb 2022 00:07:16 +0300 Subject: [PATCH 60/85] ipdoctest: Merge upstream changes to pytest doctest plugin --- IPython/testing/plugin/pytest_ipdoctest.py | 183 +++++++++++---------- 1 file changed, 93 insertions(+), 90 deletions(-) diff --git a/IPython/testing/plugin/pytest_ipdoctest.py b/IPython/testing/plugin/pytest_ipdoctest.py index 603e33c0be4..230b159b36b 100644 --- a/IPython/testing/plugin/pytest_ipdoctest.py +++ b/IPython/testing/plugin/pytest_ipdoctest.py @@ -14,6 +14,7 @@ import types import warnings from contextlib import contextmanager +from pathlib import Path from typing import Any from typing import Callable from typing import Dict @@ -28,8 +29,6 @@ from typing import TYPE_CHECKING from typing import Union -import py.path - import pytest from _pytest import outcomes from _pytest._code.code import ExceptionInfo @@ -42,6 +41,7 @@ from _pytest.fixtures import FixtureRequest from _pytest.nodes import Collector from _pytest.outcomes import OutcomeException +from _pytest.pathlib import fnmatch_ex from _pytest.pathlib import import_path from _pytest.python_api import approx from _pytest.warning_types import PytestWarning @@ -126,35 +126,38 @@ def pytest_unconfigure() -> None: def pytest_collect_file( - path: py.path.local, + file_path: Path, parent: Collector, ) -> Optional[Union["IPDoctestModule", "IPDoctestTextfile"]]: config = parent.config - if path.ext == ".py": - if config.option.ipdoctestmodules and not _is_setup_py(path): - mod: IPDoctestModule = IPDoctestModule.from_parent(parent, fspath=path) + if file_path.suffix == ".py": + if config.option.ipdoctestmodules and not any( + (_is_setup_py(file_path), _is_main_py(file_path)) + ): + mod: IPDoctestModule = IPDoctestModule.from_parent(parent, path=file_path) return mod - elif _is_ipdoctest(config, path, parent): - txt: IPDoctestTextfile = IPDoctestTextfile.from_parent(parent, fspath=path) + elif _is_ipdoctest(config, file_path, parent): + txt: IPDoctestTextfile = IPDoctestTextfile.from_parent(parent, path=file_path) return txt return None -def _is_setup_py(path: py.path.local) -> bool: - if path.basename != "setup.py": +def _is_setup_py(path: Path) -> bool: + if path.name != "setup.py": return False - contents = path.read_binary() + contents = path.read_bytes() return b"setuptools" in contents or b"distutils" in contents -def _is_ipdoctest(config: Config, path: py.path.local, parent) -> bool: - if path.ext in (".txt", ".rst") and parent.session.isinitpath(path): +def _is_ipdoctest(config: Config, path: Path, parent: Collector) -> bool: + if path.suffix in (".txt", ".rst") and parent.session.isinitpath(path): return True globs = config.getoption("ipdoctestglob") or ["test*.txt"] - for glob in globs: - if path.check(fnmatch=glob): - return True - return False + return any(fnmatch_ex(glob, path) for glob in globs) + + +def _is_main_py(path: Path) -> bool: + return path.name == "__main__.py" class ReprFailDoctest(TerminalRepr): @@ -273,7 +276,7 @@ def from_parent( # type: ignore runner: "IPDocTestRunner", dtest: "doctest.DocTest", ): - # incompatible signature due to to imposed limits on sublcass + # incompatible signature due to imposed limits on subclass """The public named constructor.""" return super().from_parent(name=name, parent=parent, runner=runner, dtest=dtest) @@ -372,61 +375,57 @@ def repr_failure( # type: ignore[override] elif isinstance(excinfo.value, MultipleDoctestFailures): failures = excinfo.value.failures - if failures is not None: - reprlocation_lines = [] - for failure in failures: - example = failure.example - test = failure.test - filename = test.filename - if test.lineno is None: - lineno = None - else: - lineno = test.lineno + example.lineno + 1 - message = type(failure).__name__ - # TODO: ReprFileLocation doesn't expect a None lineno. - reprlocation = ReprFileLocation(filename, lineno, message) # type: ignore[arg-type] - checker = _get_checker() - report_choice = _get_report_choice( - self.config.getoption("ipdoctestreport") - ) - if lineno is not None: - assert failure.test.docstring is not None - lines = failure.test.docstring.splitlines(False) - # add line numbers to the left of the error message - assert test.lineno is not None - lines = [ - "%03d %s" % (i + test.lineno + 1, x) - for (i, x) in enumerate(lines) - ] - # trim docstring error lines to 10 - lines = lines[max(example.lineno - 9, 0) : example.lineno + 1] - else: - lines = [ - "EXAMPLE LOCATION UNKNOWN, not showing all tests of that example" - ] - indent = ">>>" - for line in example.source.splitlines(): - lines.append(f"??? {indent} {line}") - indent = "..." - if isinstance(failure, doctest.DocTestFailure): - lines += checker.output_difference( - example, failure.got, report_choice - ).split("\n") - else: - inner_excinfo = ExceptionInfo(failure.exc_info) - lines += ["UNEXPECTED EXCEPTION: %s" % repr(inner_excinfo.value)] - lines += [ - x.strip("\n") - for x in traceback.format_exception(*failure.exc_info) - ] - reprlocation_lines.append((reprlocation, lines)) - return ReprFailDoctest(reprlocation_lines) - else: + if failures is None: return super().repr_failure(excinfo) - def reportinfo(self): + reprlocation_lines = [] + for failure in failures: + example = failure.example + test = failure.test + filename = test.filename + if test.lineno is None: + lineno = None + else: + lineno = test.lineno + example.lineno + 1 + message = type(failure).__name__ + # TODO: ReprFileLocation doesn't expect a None lineno. + reprlocation = ReprFileLocation(filename, lineno, message) # type: ignore[arg-type] + checker = _get_checker() + report_choice = _get_report_choice(self.config.getoption("ipdoctestreport")) + if lineno is not None: + assert failure.test.docstring is not None + lines = failure.test.docstring.splitlines(False) + # add line numbers to the left of the error message + assert test.lineno is not None + lines = [ + "%03d %s" % (i + test.lineno + 1, x) for (i, x) in enumerate(lines) + ] + # trim docstring error lines to 10 + lines = lines[max(example.lineno - 9, 0) : example.lineno + 1] + else: + lines = [ + "EXAMPLE LOCATION UNKNOWN, not showing all tests of that example" + ] + indent = ">>>" + for line in example.source.splitlines(): + lines.append(f"??? {indent} {line}") + indent = "..." + if isinstance(failure, doctest.DocTestFailure): + lines += checker.output_difference( + example, failure.got, report_choice + ).split("\n") + else: + inner_excinfo = ExceptionInfo.from_exc_info(failure.exc_info) + lines += ["UNEXPECTED EXCEPTION: %s" % repr(inner_excinfo.value)] + lines += [ + x.strip("\n") for x in traceback.format_exception(*failure.exc_info) + ] + reprlocation_lines.append((reprlocation, lines)) + return ReprFailDoctest(reprlocation_lines) + + def reportinfo(self) -> Tuple[Union["os.PathLike[str]", str], Optional[int], str]: assert self.dtest is not None - return self.fspath, self.dtest.lineno, "[ipdoctest] %s" % self.name + return self.path, self.dtest.lineno, "[ipdoctest] %s" % self.name def _get_flag_lookup() -> Dict[str, int]: @@ -474,9 +473,9 @@ def collect(self) -> Iterable[IPDoctestItem]: # Inspired by doctest.testfile; ideally we would use it directly, # but it doesn't support passing a custom checker. encoding = self.config.getini("ipdoctest_encoding") - text = self.fspath.read_text(encoding) - filename = str(self.fspath) - name = self.fspath.basename + text = self.path.read_text(encoding) + filename = str(self.path) + name = self.path.name globs = {"__name__": "__main__"} optionflags = get_optionflags(self) @@ -559,15 +558,20 @@ class MockAwareDocTestFinder(DocTestFinder): def _find_lineno(self, obj, source_lines): """Doctest code does not take into account `@property`, this - is a hackish way to fix it. + is a hackish way to fix it. https://bugs.python.org/issue17446 - https://bugs.python.org/issue17446 + Wrapped Doctests will need to be unwrapped so the correct + line number is returned. This will be reported upstream. #8796 """ if isinstance(obj, property): obj = getattr(obj, "fget", obj) + + if hasattr(obj, "__wrapped__"): + # Get the main obj in case of it being wrapped + obj = inspect.unwrap(obj) + # Type ignored because this is a private function. - return DocTestFinder._find_lineno( # type: ignore - self, + return super()._find_lineno( # type:ignore[misc] obj, source_lines, ) @@ -580,20 +584,22 @@ def _find( with _patch_unwrap_mock_aware(): # Type ignored because this is a private function. - DocTestFinder._find( # type: ignore - self, tests, obj, name, module, source_lines, globs, seen + super()._find( # type:ignore[misc] + tests, obj, name, module, source_lines, globs, seen ) - if self.fspath.basename == "conftest.py": + if self.path.name == "conftest.py": module = self.config.pluginmanager._importconftest( - self.fspath, self.config.getoption("importmode") + self.path, + self.config.getoption("importmode"), + rootpath=self.config.rootpath, ) else: try: - module = import_path(self.fspath) + module = import_path(self.path, root=self.config.rootpath) except ImportError: if self.config.getvalue("ipdoctest_ignore_import_errors"): - pytest.skip("unable to import module %r" % self.fspath) + pytest.skip("unable to import module %r" % self.path) else: raise # Uses internal doctest module parsing mechanism. @@ -665,7 +671,7 @@ class LiteralsOutputChecker(IPDoctestOutputChecker): ) def check_output(self, want: str, got: str, optionflags: int) -> bool: - if IPDoctestOutputChecker.check_output(self, want, got, optionflags): + if super().check_output(want, got, optionflags): return True allow_unicode = optionflags & _get_allow_unicode_flag() @@ -689,7 +695,7 @@ def remove_prefixes(regex: Pattern[str], txt: str) -> str: if allow_number: got = self._remove_unwanted_precision(want, got) - return IPDoctestOutputChecker.check_output(self, want, got, optionflags) + return super().check_output(want, got, optionflags) def _remove_unwanted_precision(self, want: str, got: str) -> str: wants = list(self._number_re.finditer(want)) @@ -702,13 +708,10 @@ def _remove_unwanted_precision(self, want: str, got: str) -> str: exponent: Optional[str] = w.group("exponent1") if exponent is None: exponent = w.group("exponent2") - if fraction is None: - precision = 0 - else: - precision = len(fraction) + precision = 0 if fraction is None else len(fraction) if exponent is not None: precision -= int(exponent) - if float(w.group()) == approx(float(g.group()), abs=10 ** -precision): + if float(w.group()) == approx(float(g.group()), abs=10**-precision): # They're close enough. Replace the text we actually # got with the text we want, so that it will match when we # check the string literally. From beb0b8251992ff8d2ee2c182666cb089781a382e Mon Sep 17 00:00:00 2001 From: Nikita Kniazev Date: Wed, 9 Feb 2022 01:24:16 +0300 Subject: [PATCH 61/85] ipdoctest: pytest<7 compatibility --- IPython/testing/plugin/pytest_ipdoctest.py | 83 ++++++++++++++++++++-- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/IPython/testing/plugin/pytest_ipdoctest.py b/IPython/testing/plugin/pytest_ipdoctest.py index 230b159b36b..809713d7c8e 100644 --- a/IPython/testing/plugin/pytest_ipdoctest.py +++ b/IPython/testing/plugin/pytest_ipdoctest.py @@ -142,6 +142,23 @@ def pytest_collect_file( return None +if int(pytest.__version__.split(".")[0]) < 7: + _collect_file = pytest_collect_file + + def pytest_collect_file( + path, + parent: Collector, + ) -> Optional[Union["IPDoctestModule", "IPDoctestTextfile"]]: + return _collect_file(Path(path), parent) + + _import_path = import_path + + def import_path(path, root): + import py.path + + return _import_path(py.path.local(path)) + + def _is_setup_py(path: Path) -> bool: if path.name != "setup.py": return False @@ -427,6 +444,12 @@ def reportinfo(self) -> Tuple[Union["os.PathLike[str]", str], Optional[int], str assert self.dtest is not None return self.path, self.dtest.lineno, "[ipdoctest] %s" % self.name + if int(pytest.__version__.split(".")[0]) < 7: + + @property + def path(self) -> Path: + return Path(self.fspath) + def _get_flag_lookup() -> Dict[str, int]: import doctest @@ -494,6 +517,27 @@ def collect(self) -> Iterable[IPDoctestItem]: self, name=test.name, runner=runner, dtest=test ) + if int(pytest.__version__.split(".")[0]) < 7: + + @property + def path(self) -> Path: + return Path(self.fspath) + + @classmethod + def from_parent( + cls, + parent, + *, + fspath=None, + path: Optional[Path] = None, + **kw, + ): + if path is not None: + import py.path + + fspath = py.path.local(path) + return super().from_parent(parent=parent, fspath=fspath, **kw) + def _check_all_skipped(test: "doctest.DocTest") -> None: """Raise pytest.skip() if all examples in the given DocTest have the SKIP @@ -589,11 +633,17 @@ def _find( ) if self.path.name == "conftest.py": - module = self.config.pluginmanager._importconftest( - self.path, - self.config.getoption("importmode"), - rootpath=self.config.rootpath, - ) + if int(pytest.__version__.split(".")[0]) < 7: + module = self.config.pluginmanager._importconftest( + self.path, + self.config.getoption("importmode"), + ) + else: + module = self.config.pluginmanager._importconftest( + self.path, + self.config.getoption("importmode"), + rootpath=self.config.rootpath, + ) else: try: module = import_path(self.path, root=self.config.rootpath) @@ -618,6 +668,27 @@ def _find( self, name=test.name, runner=runner, dtest=test ) + if int(pytest.__version__.split(".")[0]) < 7: + + @property + def path(self) -> Path: + return Path(self.fspath) + + @classmethod + def from_parent( + cls, + parent, + *, + fspath=None, + path: Optional[Path] = None, + **kw, + ): + if path is not None: + import py.path + + fspath = py.path.local(path) + return super().from_parent(parent=parent, fspath=fspath, **kw) + def _setup_fixtures(doctest_item: IPDoctestItem) -> FixtureRequest: """Used by IPDoctestTextfile and IPDoctestItem to setup fixture information.""" @@ -711,7 +782,7 @@ def _remove_unwanted_precision(self, want: str, got: str) -> str: precision = 0 if fraction is None else len(fraction) if exponent is not None: precision -= int(exponent) - if float(w.group()) == approx(float(g.group()), abs=10**-precision): + if float(w.group()) == approx(float(g.group()), abs=10 ** -precision): # They're close enough. Replace the text we actually # got with the text we want, so that it will match when we # check the string literally. From 31afadfb688639b1d0f72e9242144a3aadd4cde4 Mon Sep 17 00:00:00 2001 From: Nikita Kniazev Date: Wed, 9 Feb 2022 02:34:07 +0300 Subject: [PATCH 62/85] unpin pytest --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 4fbffaa1a07..2da02e4e598 100644 --- a/setup.cfg +++ b/setup.cfg @@ -63,7 +63,7 @@ qtconsole = qtconsole terminal = test = - pytest<7 + pytest pytest-asyncio testpath test_extra = @@ -72,7 +72,7 @@ test_extra = nbformat numpy>=1.19 pandas - pytest<7 + pytest testpath trio all = From b2f71a87af023177d38196dcf8c9c1a725b8cf59 Mon Sep 17 00:00:00 2001 From: luz paz Date: Wed, 9 Feb 2022 13:59:30 -0500 Subject: [PATCH 63/85] Fix various typos Found via `codespell -q 3` --- .github/workflows/downstream.yml | 2 +- IPython/core/completer.py | 4 ++-- IPython/extensions/tests/test_autoreload.py | 2 +- SECURITY.md | 2 +- appveyor.yml | 2 +- docs/source/sphinxext.rst | 2 +- tools/release_helper.sh | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 5c5adc1e0ab..309d03a2204 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -39,7 +39,7 @@ jobs: - name: Install and update Python dependencies run: | python -m pip install --upgrade -e file://$PWD#egg=ipython[test] - # we must instal IPython after ipykernel to get the right versions. + # we must install IPython after ipykernel to get the right versions. python -m pip install --upgrade --upgrade-strategy eager flaky ipyparallel python -m pip install --upgrade 'pytest<7' - name: pytest diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 9774fd5a4e7..89b55d2b475 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -1182,7 +1182,7 @@ def __init__( # This is a list of names of unicode characters that can be completed # into their corresponding unicode value. The list is large, so we - # laziliy initialize it on first use. Consuming code should access this + # lazily initialize it on first use. Consuming code should access this # attribute through the `@unicode_names` property. self._unicode_names = None @@ -2070,7 +2070,7 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, indexed. line_buffer : optional, str The current line the cursor is in, this is mostly due to legacy - reason that readline coudl only give a us the single current line. + reason that readline could only give a us the single current line. Prefer `full_text`. text : str The current "token" the cursor is in, mostly also for historical diff --git a/IPython/extensions/tests/test_autoreload.py b/IPython/extensions/tests/test_autoreload.py index a0fe725aa68..88637fbab9c 100644 --- a/IPython/extensions/tests/test_autoreload.py +++ b/IPython/extensions/tests/test_autoreload.py @@ -33,7 +33,7 @@ if platform.python_implementation() == "PyPy": pytest.skip( - "Current autoreload implementation is extremly slow on PyPy", + "Current autoreload implementation is extremely slow on PyPy", allow_module_level=True, ) diff --git a/SECURITY.md b/SECURITY.md index a4b9435a975..dc5db66e2a2 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -3,4 +3,4 @@ ## Reporting a Vulnerability All IPython and Jupyter security are handled via security@ipython.org. -You can find more informations on the Jupyter website. https://jupyter.org/security +You can find more information on the Jupyter website. https://jupyter.org/security diff --git a/appveyor.yml b/appveyor.yml index b1c0abe0b6c..7637841b2fd 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,7 +5,7 @@ matrix: environment: global: APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2022' - COLUMNS: 120 # Appveyor web viwer window width is 130 chars + COLUMNS: 120 # Appveyor web viewer window width is 130 chars matrix: - PYTHON: "C:\\Python38" diff --git a/docs/source/sphinxext.rst b/docs/source/sphinxext.rst index b2012fa62f6..093e04a90fc 100644 --- a/docs/source/sphinxext.rst +++ b/docs/source/sphinxext.rst @@ -204,7 +204,7 @@ suppress the seed line so it doesn't show up in the rendered output [0.22591016, 0.77731835], [0.0072729 , 0.34273127]]) -For more information on @supress and @doctest decorators, please refer to the end of this file in +For more information on @suppress and @doctest decorators, please refer to the end of this file in Pseudo-Decorators section. Another demonstration of multi-line input and output diff --git a/tools/release_helper.sh b/tools/release_helper.sh index a9208449f93..54114d18bb8 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -111,7 +111,7 @@ then sleep 1 echo $BLUE"Saving API to file $PREV_RELEASE"$NOR frappuccino IPython IPython.kernel IPython.lib IPython.qt IPython.lib.kernel IPython.html IPython.frontend IPython.external --save IPython-$PREV_RELEASE.json - echo $BLUE"comming back to $BRANCH"$NOR + echo $BLUE"coming back to $BRANCH"$NOR git checkout $BRANCH sleep 1 echo $BLUE"comparing ..."$NOR From 89f7392432db54236f7f82332700d3be0d2d88a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anton=20=C3=84lgmyr?= Date: Sat, 12 Feb 2022 02:52:24 +0100 Subject: [PATCH 64/85] Add support for autoformatting using yapf --- IPython/terminal/interactiveshell.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 4a46f2702cd..4498888fcc7 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -116,6 +116,20 @@ def black_reformat_handler(text_before_cursor): return formatted_text +def yapf_reformat_handler(text_before_cursor): + from yapf.yapflib import file_resources + from yapf.yapflib import yapf_api + + style_config = file_resources.GetDefaultStyleForDir(os.getcwd()) + formatted_text, was_formatted = yapf_api.FormatCode(text_before_cursor, style_config=style_config) + if was_formatted: + if not text_before_cursor.endswith("\n") and formatted_text.endswith("\n"): + formatted_text = formatted_text[:-1] + return formatted_text + else: + return text_before_cursor + + class TerminalInteractiveShell(InteractiveShell): mime_renderers = Dict().tag(config=True) @@ -185,7 +199,7 @@ def debugger_cls(self): autoformatter = Unicode( "black", - help="Autoformatter to reformat Terminal code. Can be `'black'` or `None`", + help="Autoformatter to reformat Terminal code. Can be `'black'`, `'yapf'` or `None`", allow_none=True ).tag(config=True) @@ -232,6 +246,8 @@ def _set_formatter(self, formatter): self.reformat_handler = lambda x:x elif formatter == 'black': self.reformat_handler = black_reformat_handler + elif formatter == 'yapf': + self.reformat_handler = yapf_reformat_handler else: raise ValueError From a036e1fb432b1951caa6730ea0909be5d53917ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anton=20=C3=84lgmyr?= Date: Sat, 12 Feb 2022 03:00:20 +0100 Subject: [PATCH 65/85] Fix some formatting --- IPython/terminal/interactiveshell.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 4498888fcc7..03e91dc6830 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -121,7 +121,9 @@ def yapf_reformat_handler(text_before_cursor): from yapf.yapflib import yapf_api style_config = file_resources.GetDefaultStyleForDir(os.getcwd()) - formatted_text, was_formatted = yapf_api.FormatCode(text_before_cursor, style_config=style_config) + formatted_text, was_formatted = yapf_api.FormatCode( + text_before_cursor, style_config=style_config + ) if was_formatted: if not text_before_cursor.endswith("\n") and formatted_text.endswith("\n"): formatted_text = formatted_text[:-1] From 5b6dbf75bcbf153cadf509962e95c50a3d9bd68b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anton=20=C3=84lgmyr?= Date: Sat, 12 Feb 2022 16:16:52 +0100 Subject: [PATCH 66/85] Fix darker complaint, even though inconsistent --- IPython/terminal/interactiveshell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 03e91dc6830..37773023db4 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -248,7 +248,7 @@ def _set_formatter(self, formatter): self.reformat_handler = lambda x:x elif formatter == 'black': self.reformat_handler = black_reformat_handler - elif formatter == 'yapf': + elif formatter == "yapf": self.reformat_handler = yapf_reformat_handler else: raise ValueError From 83684a4776fddbbbbcaa0a47fa3914262fc5cf2a Mon Sep 17 00:00:00 2001 From: "Samuel B. Johnson" Date: Thu, 17 Feb 2022 20:13:30 -0600 Subject: [PATCH 67/85] :bug: Remove `ls` from list of system commands to warn about --- IPython/core/interactiveshell.py | 9 ++------- IPython/core/tests/test_interactiveshell.py | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 61e15415000..a87db36d0b0 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2414,14 +2414,9 @@ def system_raw(self, cmd): cmd = self.var_expand(cmd, depth=1) # warn if there is an IPython magic alternative. main_cmd = cmd.split()[0] - has_magic_alternatives = ("pip", "conda", "cd", "ls") + has_magic_alternatives = ("pip", "conda", "cd") - # had to check if the command was an alias expanded because of `ls` - is_alias_expanded = self.alias_manager.is_alias(main_cmd) and ( - self.alias_manager.retrieve_alias(main_cmd).strip() == cmd.strip() - ) - - if main_cmd in has_magic_alternatives and not is_alias_expanded: + if main_cmd in has_magic_alternatives: warnings.warn( ( "You executed the system command !{0} which may not work " diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py index 09dbd967706..230a498cfcc 100644 --- a/IPython/core/tests/test_interactiveshell.py +++ b/IPython/core/tests/test_interactiveshell.py @@ -623,7 +623,7 @@ def test_control_c(self, *mocks): self.assertEqual(ip.user_ns["_exit_code"], -signal.SIGINT) def test_magic_warnings(self): - for magic_cmd in ("ls", "pip", "conda", "cd"): + for magic_cmd in ("pip", "conda", "cd"): with self.assertWarnsRegex(Warning, "You executed the system command"): ip.system_raw(magic_cmd) From d11e987f174a15f1640f8006c86f58d884c3faa4 Mon Sep 17 00:00:00 2001 From: Fabio Zadrozny Date: Thu, 17 Feb 2022 16:39:32 -0300 Subject: [PATCH 68/85] Set co_name for cells run line by line. Fixes https://github.com/ipython/ipykernel/issues/841 --- IPython/core/interactiveshell.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 61e15415000..a50816f181e 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -15,6 +15,7 @@ import ast import atexit import builtins as builtin_mod +import dis import functools import inspect import os @@ -3141,6 +3142,29 @@ def transform_ast(self, node): ast.fix_missing_locations(node) return node + def _update_code_co_name(self, code): + """Python 3.10 changed the behaviour so that whenever a code object + is assembled in the compile(ast) the co_firstlineno would be == 1. + + This makes pydevd/debugpy think that all cells invoked are the same + since it caches information based on (co_firstlineno, co_name, co_filename). + + Given that, this function changes the code 'co_name' to be unique + based on the first real lineno of the code (which also has a nice + side effect of customizing the name so that it's not always ). + + See: https://github.com/ipython/ipykernel/issues/841 + """ + if not hasattr(code, "replace"): + # It may not be available on older versions of Python (only + # available for 3.8 onwards). + return code + try: + first_real_line = next(dis.findlinestarts(code))[1] + except StopIteration: + return code + return code.replace(co_name="" % (first_real_line,)) + async def run_ast_nodes( self, nodelist: ListType[stmt], @@ -3239,6 +3263,7 @@ def compare(code): else 0x0 ): code = compiler(mod, cell_name, mode) + code = self._update_code_co_name(code) asy = compare(code) if await self.run_code(code, result, async_=asy): return True From 258a093e0e4fd27ca7ee24da9793609f811d2856 Mon Sep 17 00:00:00 2001 From: Nathan Buckner Date: Sun, 20 Feb 2022 08:50:13 -0800 Subject: [PATCH 69/85] Fix for symlink resolving --- IPython/core/interactiveshell.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 61e15415000..26198831992 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -772,10 +772,13 @@ def init_virtualenv(self): # stdlib venv may symlink sys.executable, so we can't use realpath. # but others can symlink *to* the venv Python, so we can't just use sys.executable. # So we just check every item in the symlink tree (generally <= 3) + current_dir = Path(os.curdir).absolute() paths = [p] while p.is_symlink(): - p = Path(os.readlink(p)) - paths.append(p.resolve()) + os.chdir(p.parent) + p = Path(os.readlink(p)).absolute() + paths.append(p) + os.chdir(current_dir) # In Cygwin paths like "c:\..." and '\cygdrive\c\...' are possible if p_venv.parts[1] == "cygdrive": From 3ad6bdca0e5f186e1205b7436668731539173111 Mon Sep 17 00:00:00 2001 From: Nathan Buckner Date: Mon, 21 Feb 2022 05:19:24 -0800 Subject: [PATCH 70/85] Removed chdir and changed to joining the path if not absolute path. This better follows the readlink docs. --- IPython/core/interactiveshell.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 26198831992..42364eb960a 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -772,13 +772,13 @@ def init_virtualenv(self): # stdlib venv may symlink sys.executable, so we can't use realpath. # but others can symlink *to* the venv Python, so we can't just use sys.executable. # So we just check every item in the symlink tree (generally <= 3) - current_dir = Path(os.curdir).absolute() paths = [p] while p.is_symlink(): - os.chdir(p.parent) - p = Path(os.readlink(p)).absolute() + new_path = p.readlink() + if not new_path.is_absolute(): + new_path = p.parent / new_path + p = new_path paths.append(p) - os.chdir(current_dir) # In Cygwin paths like "c:\..." and '\cygdrive\c\...' are possible if p_venv.parts[1] == "cygdrive": From 946e545b181fea95e876b15c51fc60a5a89bf934 Mon Sep 17 00:00:00 2001 From: Lucy McPhail Date: Mon, 21 Feb 2022 20:26:24 +0000 Subject: [PATCH 71/85] Improve auto_match for quotes Only insert a pair of quotes if there are an even number of quotes preceding the cursor. This way, if the cursor is inside an unclosed string, typing the closing quote will not insert a pair. --- IPython/terminal/shortcuts.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index ea204f8ddd8..274f9b250bb 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -139,12 +139,24 @@ def _(event): event.current_buffer.insert_text("{}") event.current_buffer.cursor_left() - @kb.add('"', filter=focused_insert & auto_match & following_text(r"[,)}\]]|$")) + @kb.add( + '"', + filter=focused_insert + & auto_match + & preceding_text(r'^([^"]+|"[^"]*")*$') + & following_text(r"[,)}\]]|$"), + ) def _(event): event.current_buffer.insert_text('""') event.current_buffer.cursor_left() - @kb.add("'", filter=focused_insert & auto_match & following_text(r"[,)}\]]|$")) + @kb.add( + "'", + filter=focused_insert + & auto_match + & preceding_text(r"^([^']+|'[^']*')*$") + & following_text(r"[,)}\]]|$"), + ) def _(event): event.current_buffer.insert_text("''") event.current_buffer.cursor_left() @@ -186,16 +198,6 @@ def _(event): event.current_buffer.insert_text("{}" + dashes) event.current_buffer.cursor_left(len(dashes) + 1) - @kb.add('"', filter=focused_insert & auto_match & preceding_text(r".*(r|R)$")) - def _(event): - event.current_buffer.insert_text('""') - event.current_buffer.cursor_left() - - @kb.add("'", filter=focused_insert & auto_match & preceding_text(r".*(r|R)$")) - def _(event): - event.current_buffer.insert_text("''") - event.current_buffer.cursor_left() - # just move cursor @kb.add(")", filter=focused_insert & auto_match & following_text(r"^\)")) @kb.add("]", filter=focused_insert & auto_match & following_text(r"^\]")) From 1cf66263c96b85749216150d72bf644d6349ed2d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Feb 2022 10:59:58 +0100 Subject: [PATCH 72/85] doc parssing issue --- IPython/utils/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/utils/text.py b/IPython/utils/text.py index a1754d254de..ef75f9331d7 100644 --- a/IPython/utils/text.py +++ b/IPython/utils/text.py @@ -471,7 +471,7 @@ def strip_ansi(source): class EvalFormatter(Formatter): """A String Formatter that allows evaluation of simple expressions. - Note that this version interprets a : as specifying a format string (as per + Note that this version interprets a `:` as specifying a format string (as per standard string formatting), so if slicing is required, you must explicitly create a slice. From bce649cba874ba7965ac894564e67e446a7854ff Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Feb 2022 11:17:06 +0100 Subject: [PATCH 73/85] formatting --- IPython/core/interactiveshell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 17aacfc3013..9012f0f20a2 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -3680,7 +3680,7 @@ def atexit_operations(self): del self.tempdirs # Restore user's cursor - if hasattr(self, "editing_mode") and self.editing_mode == 'vi': + if hasattr(self, "editing_mode") and self.editing_mode == "vi": sys.stdout.write("\x1b[0 q") sys.stdout.flush() From 422f8d9cf8c3858aab934853780a872610df50e4 Mon Sep 17 00:00:00 2001 From: Can Sarigol Date: Mon, 24 Jan 2022 08:44:18 +0100 Subject: [PATCH 74/85] Get history file from shell to debugger if it exists. --- IPython/terminal/debugger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/IPython/terminal/debugger.py b/IPython/terminal/debugger.py index d76550d878f..8448d96370d 100644 --- a/IPython/terminal/debugger.py +++ b/IPython/terminal/debugger.py @@ -68,6 +68,8 @@ def gen_comp(self, text): self.debugger_history = FileHistory(os.path.expanduser(str(p))) else: self.debugger_history = InMemoryHistory() + else: + self.debugger_history = self.shell.debugger_history options = dict( message=(lambda: PygmentsTokens(get_prompt_tokens())), From a9b523c7047fe12c49373972c6b092ed5fc29e99 Mon Sep 17 00:00:00 2001 From: Thomas Nicholas Date: Wed, 16 Feb 2022 16:03:03 -0500 Subject: [PATCH 75/85] match only pseudo-decorators --- IPython/sphinxext/ipython_directive.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/IPython/sphinxext/ipython_directive.py b/IPython/sphinxext/ipython_directive.py index ac0964032a5..093fd7a362c 100644 --- a/IPython/sphinxext/ipython_directive.py +++ b/IPython/sphinxext/ipython_directive.py @@ -220,6 +220,8 @@ # for tokenizing blocks COMMENT, INPUT, OUTPUT = range(3) +PSEUDO_DECORATORS = ["suppress", "verbatim", "savefig", "doctest"] + #----------------------------------------------------------------------------- # Functions and class declarations #----------------------------------------------------------------------------- @@ -263,11 +265,14 @@ def block_parser(part, rgxin, rgxout, fmtin, fmtout): block.append((COMMENT, line)) continue - if line_stripped.startswith('@'): - # Here is where we assume there is, at most, one decorator. - # Might need to rethink this. - decorator = line_stripped - continue + if any( + line_stripped.startswith('@' + pseudo_decorator) for pseudo_decorator in PSEUDO_DECORATORS + ): + if decorator: + raise RuntimeError("Applying multiple pseudo-decorators on one line is not supported") + else: + decorator = line_stripped + continue # does this look like an input line? matchin = rgxin.match(line) From 5fec429e39671a5c359e6fc17ab59ece4e139847 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Feb 2022 11:27:37 +0100 Subject: [PATCH 76/85] reformatting --- IPython/sphinxext/ipython_directive.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/IPython/sphinxext/ipython_directive.py b/IPython/sphinxext/ipython_directive.py index 093fd7a362c..18bdfcae993 100644 --- a/IPython/sphinxext/ipython_directive.py +++ b/IPython/sphinxext/ipython_directive.py @@ -266,10 +266,13 @@ def block_parser(part, rgxin, rgxout, fmtin, fmtout): continue if any( - line_stripped.startswith('@' + pseudo_decorator) for pseudo_decorator in PSEUDO_DECORATORS - ): + line_stripped.startswith("@" + pseudo_decorator) + for pseudo_decorator in PSEUDO_DECORATORS + ): if decorator: - raise RuntimeError("Applying multiple pseudo-decorators on one line is not supported") + raise RuntimeError( + "Applying multiple pseudo-decorators on one line is not supported" + ) else: decorator = line_stripped continue From f6e3393f88b8d188726220c72e775a7a26c3249b Mon Sep 17 00:00:00 2001 From: ygeyzel Date: Sat, 22 Jan 2022 17:24:56 +0200 Subject: [PATCH 77/85] Typing '%' restrict autocompletion to magics --- IPython/core/completer.py | 5 +++-- IPython/core/tests/test_completer.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 89b55d2b475..0579e684e81 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -2139,8 +2139,9 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, # different types of objects. The rlcomplete() method could then # simply collapse the dict into a list for readline, but we'd have # richer completion semantics in other environments. - completions:Iterable[Any] = [] - if self.use_jedi: + is_magic_prefix = len(text) > 0 and text[0] == "%" + completions: Iterable[Any] = [] + if self.use_jedi and not is_magic_prefix: if not full_text: full_text = line_buffer completions = self._jedi_matches( diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 5f791e84f83..746a1e68261 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -1262,3 +1262,14 @@ def meth_2(self, meth2_arg1, meth2_arg2): _, matches = ip.complete(None, "test.meth(") self.assertIn("meth_arg1=", matches) self.assertNotIn("meth2_arg1=", matches) + + def test_percent_symbol_restrict_to_magic_completions(self): + ip = get_ipython() + completer = ip.Completer + text = "%a" + + with provisionalcompleter(): + completer.use_jedi = True + completions = completer.completions(text, len(text)) + for c in completions: + self.assertEqual(c.text[0], "%") From 5ccd62a38c58306166898a417e115707e3e673fe Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Sat, 15 Jan 2022 10:09:31 +0000 Subject: [PATCH 78/85] Revert "enable formatting by default" This reverts PR #13397 / commit df3248f00a60859ed07f81e87531856b30891396, whilst leaving in place the internal improvements. --- IPython/terminal/interactiveshell.py | 2 +- docs/source/whatsnew/version8.rst | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 37773023db4..212692ab6d1 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -200,7 +200,7 @@ def debugger_cls(self): ).tag(config=True) autoformatter = Unicode( - "black", + None, help="Autoformatter to reformat Terminal code. Can be `'black'`, `'yapf'` or `None`", allow_none=True ).tag(config=True) diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index 7b2318f3dcd..12c5d1f17c7 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -344,12 +344,11 @@ For more information please see the following unit test : ``extensions/tests/tes Auto formatting with black in the CLI ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If ``black`` is installed in the same environment as IPython, terminal IPython -will now *by default* reformat the code in the CLI when possible. You can -disable this with ``--TerminalInteractiveShell.autoformatter=None``. - This feature was present in 7.x, but disabled by default. +In 8.0, input was automatically reformatted with Black when black was installed. +This feature has been reverted for the time being. +You can re-enable it by setting ``TerminalInteractiveShell.autoformatter`` to ``"black"`` History Range Glob feature ~~~~~~~~~~~~~~~~~~~~~~~~~~ From 96cbe589e4b201af115a443bcaa3bab6f9fcc697 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Feb 2022 11:39:18 +0100 Subject: [PATCH 79/85] Remove IPython/lib/security. This should be part of Jupyter notebook / jupyter lab, and has no reason to stay here. It might even be dangerous as it is not maintained here. --- IPython/lib/__init__.py | 10 --- IPython/lib/security.py | 114 ----------------------------- IPython/lib/tests/test_security.py | 27 ------- 3 files changed, 151 deletions(-) delete mode 100644 IPython/lib/security.py delete mode 100644 IPython/lib/tests/test_security.py diff --git a/IPython/lib/__init__.py b/IPython/lib/__init__.py index 8eb89012df1..94b8ade4ec9 100644 --- a/IPython/lib/__init__.py +++ b/IPython/lib/__init__.py @@ -9,13 +9,3 @@ # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. #----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -from IPython.lib.security import passwd - -#----------------------------------------------------------------------------- -# Code -#----------------------------------------------------------------------------- diff --git a/IPython/lib/security.py b/IPython/lib/security.py deleted file mode 100644 index 152561dabad..00000000000 --- a/IPython/lib/security.py +++ /dev/null @@ -1,114 +0,0 @@ -""" -Password generation for the IPython notebook. -""" -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- -# Stdlib -import getpass -import hashlib -import random - -# Our own -from IPython.core.error import UsageError -from IPython.utils.py3compat import encode - -#----------------------------------------------------------------------------- -# Globals -#----------------------------------------------------------------------------- - -# Length of the salt in nr of hex chars, which implies salt_len * 4 -# bits of randomness. -salt_len = 12 - -#----------------------------------------------------------------------------- -# Functions -#----------------------------------------------------------------------------- - -def passwd(passphrase=None, algorithm='sha1'): - """Generate hashed password and salt for use in notebook configuration. - - In the notebook configuration, set `c.NotebookApp.password` to - the generated string. - - Parameters - ---------- - passphrase : str - Password to hash. If unspecified, the user is asked to input - and verify a password. - algorithm : str - Hashing algorithm to use (e.g, 'sha1' or any argument supported - by :func:`hashlib.new`). - - Returns - ------- - hashed_passphrase : str - Hashed password, in the format 'hash_algorithm:salt:passphrase_hash'. - - Examples - -------- - >>> passwd('mypassword') - 'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12' # random - - """ - if passphrase is None: - for i in range(3): - p0 = getpass.getpass('Enter password: ') - p1 = getpass.getpass('Verify password: ') - if p0 == p1: - passphrase = p0 - break - else: - print('Passwords do not match.') - else: - raise UsageError('No matching passwords found. Giving up.') - - h = hashlib.new(algorithm) - salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len) - h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii')) - - return ':'.join((algorithm, salt, h.hexdigest())) - - -def passwd_check(hashed_passphrase, passphrase): - """Verify that a given passphrase matches its hashed version. - - Parameters - ---------- - hashed_passphrase : str - Hashed password, in the format returned by `passwd`. - passphrase : str - Passphrase to validate. - - Returns - ------- - valid : bool - True if the passphrase matches the hash. - - Examples - -------- - >>> from IPython.lib.security import passwd_check - >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a', - ... 'mypassword') - True - - >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a', - ... 'anotherpassword') - False - """ - try: - algorithm, salt, pw_digest = hashed_passphrase.split(':', 2) - except (ValueError, TypeError): - return False - - try: - h = hashlib.new(algorithm) - except ValueError: - return False - - if len(pw_digest) == 0: - return False - - h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii')) - - return h.hexdigest() == pw_digest diff --git a/IPython/lib/tests/test_security.py b/IPython/lib/tests/test_security.py deleted file mode 100644 index 27c32ab7328..00000000000 --- a/IPython/lib/tests/test_security.py +++ /dev/null @@ -1,27 +0,0 @@ -# coding: utf-8 -from IPython.lib import passwd -from IPython.lib.security import passwd_check, salt_len - -def test_passwd_structure(): - p = passwd("passphrase") - algorithm, salt, hashed = p.split(":") - assert algorithm == "sha1" - assert len(salt) == salt_len - assert len(hashed) == 40 - -def test_roundtrip(): - p = passwd("passphrase") - assert passwd_check(p, "passphrase") is True - - -def test_bad(): - p = passwd('passphrase') - assert passwd_check(p, p) is False - assert passwd_check(p, "a:b:c:d") is False - assert passwd_check(p, "a:b") is False - - -def test_passwd_check_unicode(): - # GH issue #4524 - phash = u'sha1:23862bc21dd3:7a415a95ae4580582e314072143d9c382c491e4f' - assert passwd_check(phash, u"łe¶ŧ←↓→") From 44d64dc36e727cde56d7b1986ee52228ddde1af9 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Feb 2022 11:56:58 +0100 Subject: [PATCH 80/85] Update some deprecated ip.magic() to run_line-magic in tests --- IPython/core/tests/test_alias.py | 8 ++-- IPython/core/tests/test_handlers.py | 64 ++++++++++++++++------------- IPython/core/tests/test_history.py | 12 +++--- 3 files changed, 45 insertions(+), 39 deletions(-) diff --git a/IPython/core/tests/test_alias.py b/IPython/core/tests/test_alias.py index a84b0095334..32d2e2f711e 100644 --- a/IPython/core/tests/test_alias.py +++ b/IPython/core/tests/test_alias.py @@ -43,11 +43,11 @@ def test_alias_args_error(): def test_alias_args_commented(): """Check that alias correctly ignores 'commented out' args""" - _ip.magic('alias commetarg echo this is %%s a commented out arg') - + _ip.run_line_magic("alias", "commentarg echo this is %%s a commented out arg") + with capture_output() as cap: - _ip.run_cell('commetarg') - + _ip.run_cell("commentarg") + # strip() is for pytest compat; testing via iptest patch IPython shell # in testing.globalipapp and replace the system call which messed up the # \r\n diff --git a/IPython/core/tests/test_handlers.py b/IPython/core/tests/test_handlers.py index e151e70ee91..604dadee1ab 100644 --- a/IPython/core/tests/test_handlers.py +++ b/IPython/core/tests/test_handlers.py @@ -56,36 +56,42 @@ def test_handlers(): ip.user_ns['autocallable'] = autocallable # auto - ip.magic('autocall 0') + ip.run_line_magic("autocall", "0") # Only explicit escapes or instances of IPyAutocallable should get # expanded - run([ - ('len "abc"', 'len "abc"'), - ('autocallable', 'autocallable()'), - # Don't add extra brackets (gh-1117) - ('autocallable()', 'autocallable()'), - ]) - ip.magic('autocall 1') - run([ - ('len "abc"', 'len("abc")'), - ('len "abc";', 'len("abc");'), # ; is special -- moves out of parens - # Autocall is turned off if first arg is [] and the object - # is both callable and indexable. Like so: - ('len [1,2]', 'len([1,2])'), # len doesn't support __getitem__... - ('call_idx [1]', 'call_idx [1]'), # call_idx *does*.. - ('call_idx 1', 'call_idx(1)'), - ('len', 'len'), # only at 2 does it auto-call on single args - ]) - ip.magic('autocall 2') - run([ - ('len "abc"', 'len("abc")'), - ('len "abc";', 'len("abc");'), - ('len [1,2]', 'len([1,2])'), - ('call_idx [1]', 'call_idx [1]'), - ('call_idx 1', 'call_idx(1)'), - # This is what's different: - ('len', 'len()'), # only at 2 does it auto-call on single args - ]) - ip.magic('autocall 1') + run( + [ + ('len "abc"', 'len "abc"'), + ("autocallable", "autocallable()"), + # Don't add extra brackets (gh-1117) + ("autocallable()", "autocallable()"), + ] + ) + ip.run_line_magic("autocall", "1") + run( + [ + ('len "abc"', 'len("abc")'), + ('len "abc";', 'len("abc");'), # ; is special -- moves out of parens + # Autocall is turned off if first arg is [] and the object + # is both callable and indexable. Like so: + ("len [1,2]", "len([1,2])"), # len doesn't support __getitem__... + ("call_idx [1]", "call_idx [1]"), # call_idx *does*.. + ("call_idx 1", "call_idx(1)"), + ("len", "len"), # only at 2 does it auto-call on single args + ] + ) + ip.run_line_magic("autocall", "2") + run( + [ + ('len "abc"', 'len("abc")'), + ('len "abc";', 'len("abc");'), + ("len [1,2]", "len([1,2])"), + ("call_idx [1]", "call_idx [1]"), + ("call_idx 1", "call_idx(1)"), + # This is what's different: + ("len", "len()"), # only at 2 does it auto-call on single args + ] + ) + ip.run_line_magic("autocall", "1") assert failures == [] diff --git a/IPython/core/tests/test_history.py b/IPython/core/tests/test_history.py index 388ebc64acd..73d50c87d34 100644 --- a/IPython/core/tests/test_history.py +++ b/IPython/core/tests/test_history.py @@ -52,13 +52,13 @@ def test_history(): # Check whether specifying a range beyond the end of the current # session results in an error (gh-804) - ip.magic('%hist 2-500') + ip.run_line_magic("hist", "2-500") # Check that we can write non-ascii characters to a file - ip.magic("%%hist -f %s" % (tmp_path / "test1")) - ip.magic("%%hist -pf %s" % (tmp_path / "test2")) - ip.magic("%%hist -nf %s" % (tmp_path / "test3")) - ip.magic("%%save %s 1-10" % (tmp_path / "test4")) + ip.run_line_magic("hist", "-f %s" % (tmp_path / "test1")) + ip.run_line_magic("hist", "-pf %s" % (tmp_path / "test2")) + ip.run_line_magic("hist", "-nf %s" % (tmp_path / "test3")) + ip.run_line_magic("save", "%s 1-10" % (tmp_path / "test4")) # New session ip.history_manager.reset() @@ -124,7 +124,7 @@ def test_history(): # Cross testing: check that magic %save can get previous session. testfilename = (tmp_path / "test.py").resolve() - ip.magic("save " + str(testfilename) + " ~1/1-3") + ip.run_line_magic("save", str(testfilename) + " ~1/1-3") with io.open(testfilename, encoding="utf-8") as testfile: assert testfile.read() == "# coding: utf-8\n" + "\n".join(hist) + "\n" From 88897b13403f10e426cd4fb5a5d5d3e205cb8b92 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Feb 2022 12:39:35 +0100 Subject: [PATCH 81/85] Try to avoid network connection during tests. Mitigate #13468 --- IPython/core/display.py | 4 ++-- IPython/core/magics/osm.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/IPython/core/display.py b/IPython/core/display.py index d36a176c3bf..933295ad6ce 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -884,7 +884,7 @@ def __init__( a URL, or a filename from which to load image data. The result is always embedding image data for inline images. - >>> Image('http://www.google.fr/images/srpr/logo3w.png') + >>> Image('https://www.google.fr/images/srpr/logo3w.png') # doctest: +SKIP >>> Image('/path/to/image.jpg') @@ -897,7 +897,7 @@ def __init__( it only generates ```` tag with a link to the source. This will not work in the qtconsole or offline. - >>> Image(url='http://www.google.fr/images/srpr/logo3w.png') + >>> Image(url='https://www.google.fr/images/srpr/logo3w.png') """ diff --git a/IPython/core/magics/osm.py b/IPython/core/magics/osm.py index 11fec812ef6..42ed876ed15 100644 --- a/IPython/core/magics/osm.py +++ b/IPython/core/magics/osm.py @@ -127,7 +127,7 @@ def alias(self, parameter_s=''): Aliases expand Python variables just like system calls using ! or !! do: all expressions prefixed with '$' get expanded. For details of the semantic rules, see PEP-215: - http://www.python.org/peps/pep-0215.html. This is the library used by + https://www.python.org/dev/peps/pep-0215/. This is the library used by IPython for variable expansion. If you want to access a true shell variable, an extra $ is necessary to prevent its expansion by IPython:: From ccfe7e7389055919817f4acb60f58c3e926a8519 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Feb 2022 14:31:28 +0100 Subject: [PATCH 82/85] Whats new 7.32 --- docs/source/whatsnew/version7.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 6597fe555f8..331db654b58 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -9,6 +9,10 @@ IPython 7.32 ============ + +Autoload magic lazily +--------------------- + The ability to configure magics to be lazily loaded has been added to IPython. See the ``ipython --help-all`` section on ``MagicsManager.lazy_magic``. One can now use:: @@ -21,6 +25,21 @@ One can now use:: And on first use of ``%my_magic``, or corresponding cell magic, or other line magic, the corresponding ``load_ext`` will be called just before trying to invoke the magic. +Misc +---- + + - Update sphinxify for Docrepr 0.2.0 :ghpull:`13503`. + - Set co_name for cells run line by line (to fix debugging with Python 3.10) + :ghpull:`13535` + + +Many thanks to all the contributors to this release. You can find all individual +contributions to this milestone `on github +`__. + +Thanks as well to the `D. E. Shaw group `__ for sponsoring +work on IPython and related libraries. + .. _version 7.31: IPython 7.31 From 218264979f050dcdacd8964e16de79154f466cb6 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Feb 2022 14:24:36 +0100 Subject: [PATCH 83/85] wn 8 --- docs/source/whatsnew/version8.rst | 71 +++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index 12c5d1f17c7..02544df0ebe 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -3,6 +3,76 @@ ============ +.. _version 8.1: + +IPython 8.1.0 +------------- + +IPython 8.1 is the first minor release after 8.0 and fixes a number of bugs and +Update a few behavior that were problematic with the 8.0 as with many new major +release. + +Note that beyond the changes listed here, IPython 8.1.0 also contains all the +features listed in :ref:`version 7.32`. + + - Misc and multiple fixes around quotation auto-closing. It is now disabled by + default. Run with ``TerminalInteractiveShell.auto_match=True`` to re-enabled + - Require pygments>=2.4.0 :ghpull:`13459`, this was implicit in the code, but + is now explicit in ``setup.cfg``/``setup.py`` + - Docs improvement of ``core.magic_arguments`` examples. :ghpull:`13433` + - Multi-line edit executes too early with await. :ghpull:`13424` + + - ``black`` is back as an optional dependency, and autoformatting disabled by + default until some fixes are implemented (black improperly reformat magics). + :ghpull:`13471` Additionally the ability to use ``yapf`` as a code + reformatter has been added :ghpull:`13528` . You can use + ``TerminalInteractiveShell.autoformatter="black"``, + ``TerminalInteractiveShell.autoformatter="yapf"`` to re-enable auto formating + with black, or switch to yapf. + + - Fix and issue where ``display`` was not defined. + + - Auto suggestions are now configurable. Currently only + ``AutoSuggestFromHistory`` (default) and ``None``. new provider contribution + welcomed. :ghpull:`13475` + + - multiple packaging/testing improvement to simplify downstream packaging + (xfail with reasons, try to not access network...). + + - Update deprecation. ``InteractiveShell.magic`` internal method has been + deprecated for many years but did not emit a warning until now. + + - internal ``appended_to_syspath`` context manager has been deprecated. + + - fix an issue with symlinks in virtualenv :ghpull:`13537` + + - Fix an issue with vim mode, where cursor would not be reset on exit :ghpull:`13472` + + - ipython directive now remove only known pseudo-decorators :ghpull:`13532` + + - ``IPython/lib/security`` which used to be used for jupyter notebook has been + removed. + + - Fix an issue where ``async with`` would execute on new lines. :ghpull:`13436` + + +We want to remind users that IPython is part of the Jupyter organisations, and +thus governed by a Code of Conduct. Some of the behavior we have seen on GitHub is not acceptable. +Abuse and non-respectful comments on discussion will not be tolerated. + +Many thanks to all the contributors to this release, many of the above fixed issue and +new features where done by first time contributors, showing there is still +plenty of easy contribution possible in IPython +. You can find all individual contributions +to this milestone `on github `__. + +Thanks as well to the `D. E. Shaw group `__ for sponsoring +work on IPython and related libraries. In particular the Lazy autoloading of +magics that you will find described in the 7.32 release notes. + + +.. _version 8.0.1: + IPython 8.0.1 (CVE-2022-21699) ------------------------------ @@ -45,6 +115,7 @@ Thus starting with this version: Further details can be read on the `GitHub Advisory `__ +.. _version 8.0: IPython 8.0 ----------- From e5d78c598bcd615b1007c41508a2486a97598a97 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Feb 2022 15:12:07 +0100 Subject: [PATCH 84/85] Increase test coverage --- IPython/core/tests/test_interactiveshell.py | 50 ++++++++++----------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py index 230a498cfcc..10cce1fd646 100644 --- a/IPython/core/tests/test_interactiveshell.py +++ b/IPython/core/tests/test_interactiveshell.py @@ -380,7 +380,8 @@ def test_ofind_property_with_error(self): class A(object): @property def foo(self): - raise NotImplementedError() + raise NotImplementedError() # pragma: no cover + a = A() found = ip._ofind('a.foo', [('locals', locals())]) @@ -392,7 +393,7 @@ def test_ofind_multiple_attribute_lookups(self): class A(object): @property def foo(self): - raise NotImplementedError() + raise NotImplementedError() # pragma: no cover a = A() a.a = A() @@ -585,9 +586,9 @@ def test_exit_code_signal(self): self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGALRM) @onlyif_cmds_exist("csh") - def test_exit_code_signal_csh(self): - SHELL = os.environ.get('SHELL', None) - os.environ['SHELL'] = find_cmd("csh") + def test_exit_code_signal_csh(self): # pragma: no cover + SHELL = os.environ.get("SHELL", None) + os.environ["SHELL"] = find_cmd("csh") try: self.test_exit_code_signal() finally: @@ -615,7 +616,7 @@ def test_1(self): def test_control_c(self, *mocks): try: self.system("sleep 1 # wont happen") - except KeyboardInterrupt: + except KeyboardInterrupt: # pragma: no cove self.fail( "system call should intercept " "keyboard interrupt from subprocess.call" @@ -679,16 +680,20 @@ def setUp(self): def tearDown(self): ip.ast_transformers.remove(self.negator) - + + def test_non_int_const(self): + with tt.AssertPrints("hello"): + ip.run_cell('print("hello")') + def test_run_cell(self): - with tt.AssertPrints('-34'): - ip.run_cell('print (12 + 22)') - + with tt.AssertPrints("-34"): + ip.run_cell("print(12 + 22)") + # A named reference to a number shouldn't be transformed. - ip.user_ns['n'] = 55 - with tt.AssertNotPrints('-55'): - ip.run_cell('print (n)') - + ip.user_ns["n"] = 55 + with tt.AssertNotPrints("-55"): + ip.run_cell("print(n)") + def test_timeit(self): called = set() def f(x): @@ -796,7 +801,11 @@ def test_run_cell(self): # This shouldn't throw an error ip.run_cell("o = 2.0") self.assertEqual(ip.user_ns['o'], 2.0) - + + def test_run_cell_non_int(self): + ip.run_cell("n = 'a'") + assert self.calls == [] + def test_timeit(self): called = set() def f(x): @@ -815,14 +824,9 @@ def f(x): class ErrorTransformer(ast.NodeTransformer): """Throws an error when it sees a number.""" - # for Python 3.7 and earlier - def visit_Num(self, node): - raise ValueError("test") - - # for Python 3.8+ def visit_Constant(self, node): if isinstance(node.value, int): - return self.visit_Num(node) + raise ValueError("test") return node @@ -845,10 +849,6 @@ class StringRejector(ast.NodeTransformer): not be executed by throwing an InputRejected. """ - #for python 3.7 and earlier - def visit_Str(self, node): - raise InputRejected("test") - # 3.8 only def visit_Constant(self, node): if isinstance(node.value, str): From bd470fe8cea36d6b1c65820e3e9fdb26d26ad422 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Feb 2022 16:08:21 +0100 Subject: [PATCH 85/85] release 8.1.0 --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index c1c90b13bf2..d67ef21018c 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,7 +20,7 @@ _version_patch = 0 _version_extra = ".dev" # _version_extra = "rc1" -# _version_extra = "" # Uncomment this for full releases +_version_extra = "" # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch]