From 8e8a8f15970822a6242dc2ecba2ba4947204b4bb Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 9 Jun 2020 21:21:40 -0700 Subject: [PATCH 1/7] MAINT: make typing module available at runtime Closes https://github.com/numpy/numpy/issues/16550. This makes `np.typing.ArrayLike` and `np.typing.DtypeLike` available at runtime in addition to typing time. Some things to consider: - `ArrayLike` uses protocols, which are only in the standard library in 3.8+, but are backported in `typing_extensions`. This conditionally imports `Protocol` and sets `_SupportsArray` to `Any` at runtime if the module is not available to prevent NumPy from having a hard dependency on `typing_extensions`. Since e.g. mypy already includes `typing_extensions` as a dependency, anybody actually doing type checking will have it set correctly. - We are starting to hit the edges of "the fiction of the stubs". In particular, they could just cram everything into `__init__.pyi` and ignore the real structure of NumPy. But now that typing is available a runtime, we have to e.g. carefully import `ndarray` from `numpy` in the typing module and not from `..core.multiarray`, because otherwise mypy will think you are talking about a different ndarray. We will probably need to do some shuffling the stubs into more fitting locations to mitigate weirdness like this. --- numpy/setup.py | 1 + numpy/tests/test_public_api.py | 3 ++- numpy/tests/typing/fail/array_like.py | 8 +----- numpy/tests/typing/pass/array_like.py | 10 ++------ numpy/typing/__init__.py | 3 +++ numpy/typing/_array_like.py | 27 +++++++++++++++++++++ numpy/{typing.pyi => typing/_dtype_like.py} | 26 +++----------------- numpy/typing/_shape.py | 6 +++++ 8 files changed, 46 insertions(+), 38 deletions(-) create mode 100644 numpy/typing/__init__.py create mode 100644 numpy/typing/_array_like.py rename numpy/{typing.pyi => typing/_dtype_like.py} (68%) create mode 100644 numpy/typing/_shape.py diff --git a/numpy/setup.py b/numpy/setup.py index c6498d101560..cbf633504db3 100644 --- a/numpy/setup.py +++ b/numpy/setup.py @@ -17,6 +17,7 @@ def configuration(parent_package='',top_path=None): config.add_subpackage('polynomial') config.add_subpackage('random') config.add_subpackage('testing') + config.add_subpackage('typing') config.add_data_dir('doc') config.add_data_files('py.typed') config.add_data_files('*.pyi') diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index 7ce74bc43ef0..beaf38e5aa5d 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -98,7 +98,7 @@ def test_dir_testing(): """Assert that output of dir has only one "testing/tester" attribute without duplicate""" assert len(dir(np)) == len(set(dir(np))) - + def test_numpy_linalg(): bad_results = check_dir(np.linalg) @@ -176,6 +176,7 @@ def test_NPY_NO_EXPORT(): "polynomial.polyutils", "random", "testing", + "typing", "version", ]] diff --git a/numpy/tests/typing/fail/array_like.py b/numpy/tests/typing/fail/array_like.py index a5ef5795f3de..a97e72dc73a9 100644 --- a/numpy/tests/typing/fail/array_like.py +++ b/numpy/tests/typing/fail/array_like.py @@ -1,11 +1,5 @@ -from typing import Any, TYPE_CHECKING - import numpy as np - -if TYPE_CHECKING: - from numpy.typing import ArrayLike -else: - ArrayLike = Any +from numpy.typing import ArrayLike class A: diff --git a/numpy/tests/typing/pass/array_like.py b/numpy/tests/typing/pass/array_like.py index 098149c4b7e3..e668b496346b 100644 --- a/numpy/tests/typing/pass/array_like.py +++ b/numpy/tests/typing/pass/array_like.py @@ -1,13 +1,7 @@ -from typing import Any, List, Optional, TYPE_CHECKING +from typing import Any, List, Optional import numpy as np - -if TYPE_CHECKING: - from numpy.typing import ArrayLike, DtypeLike, _SupportsArray -else: - ArrayLike = Any - DtypeLike = Any - _SupportsArray = Any +from numpy.typing import ArrayLike, DtypeLike, _SupportsArray x1: ArrayLike = True x2: ArrayLike = 5 diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py new file mode 100644 index 000000000000..94f76a91f46c --- /dev/null +++ b/numpy/typing/__init__.py @@ -0,0 +1,3 @@ +from ._array_like import _SupportsArray, ArrayLike +from ._shape import _Shape, _ShapeLike +from ._dtype_like import DtypeLike diff --git a/numpy/typing/_array_like.py b/numpy/typing/_array_like.py new file mode 100644 index 000000000000..54a612fb449c --- /dev/null +++ b/numpy/typing/_array_like.py @@ -0,0 +1,27 @@ +import sys +from typing import Any, overload, Sequence, Tuple, Union + +from numpy import ndarray +from ._dtype_like import DtypeLike + +if sys.version_info >= (3, 8): + from typing import Protocol + HAVE_PROTOCOL = True +else: + try: + from typing_extensions import Protocol + except ImportError: + HAVE_PROTOCOL = False + else: + HAVE_PROTOCOL = True + +if HAVE_PROTOCOL: + class _SupportsArray(Protocol): + @overload + def __array__(self, __dtype: DtypeLike = ...) -> ndarray: ... + @overload + def __array__(self, dtype: DtypeLike = ...) -> ndarray: ... +else: + _SupportsArray = Any + +ArrayLike = Union[bool, int, float, complex, _SupportsArray, Sequence] diff --git a/numpy/typing.pyi b/numpy/typing/_dtype_like.py similarity index 68% rename from numpy/typing.pyi rename to numpy/typing/_dtype_like.py index f5705192a851..b9df0af04e8f 100644 --- a/numpy/typing.pyi +++ b/numpy/typing/_dtype_like.py @@ -1,17 +1,7 @@ -import sys -from typing import Any, Dict, List, overload, Sequence, Text, Tuple, Union +from typing import Any, Dict, List, Sequence, Tuple, Union -from numpy import dtype, ndarray - -if sys.version_info >= (3, 8): - from typing import Protocol -else: - from typing_extensions import Protocol - -_Shape = Tuple[int, ...] - -# Anything that can be coerced to a shape tuple -_ShapeLike = Union[int, Sequence[int]] +from numpy import dtype +from ._shape import _ShapeLike _DtypeLikeNested = Any # TODO: wait for support for recursive types @@ -45,7 +35,7 @@ Sequence[str], # names Sequence[_DtypeLikeNested], # formats Sequence[int], # offsets - Sequence[Union[bytes, Text, None]], # titles + Sequence[Union[bytes, str, None]], # titles int, # itemsize ], ], @@ -54,11 +44,3 @@ # (base_dtype, new_dtype) Tuple[_DtypeLikeNested, _DtypeLikeNested], ] - -class _SupportsArray(Protocol): - @overload - def __array__(self, __dtype: DtypeLike = ...) -> ndarray: ... - @overload - def __array__(self, dtype: DtypeLike = ...) -> ndarray: ... - -ArrayLike = Union[bool, int, float, complex, _SupportsArray, Sequence] diff --git a/numpy/typing/_shape.py b/numpy/typing/_shape.py new file mode 100644 index 000000000000..4629046ea3eb --- /dev/null +++ b/numpy/typing/_shape.py @@ -0,0 +1,6 @@ +from typing import Sequence, Tuple, Union + +_Shape = Tuple[int, ...] + +# Anything that can be coerced to a shape tuple +_ShapeLike = Union[int, Sequence[int]] From 70130f848b7c526862fa6ff9667f078a628d86a1 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 10 Jun 2020 20:48:36 -0700 Subject: [PATCH 2/7] DOC: add documentation for the numpy.typing module --- doc/source/reference/index.rst | 1 + doc/source/reference/typing.rst | 2 ++ numpy/typing/__init__.py | 53 +++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 doc/source/reference/typing.rst diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst index 2e1dcafa2a5e..661a08ffa756 100644 --- a/doc/source/reference/index.rst +++ b/doc/source/reference/index.rst @@ -22,6 +22,7 @@ For learning how to use NumPy, see the :ref:`complete documentation Date: Thu, 11 Jun 2020 08:42:55 -0700 Subject: [PATCH 3/7] DOC: add warning about typing-extensions module to numpy.typing docs Typing `ArrayLike` correctly relies on `Protocol`, so warn users that they should be on 3.8+ or install `typing-extensions` if they want everything to work as expected. --- numpy/typing/__init__.py | 9 +++++++++ numpy/typing/_array_like.py | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py index bc3442a15200..1d377f8c46e3 100644 --- a/numpy/typing/__init__.py +++ b/numpy/typing/__init__.py @@ -3,6 +3,13 @@ Typing (:mod:`numpy.typing`) ============================ +.. warning:: + + Some of the types in this module rely on features only present in + the standard library in Python 3.8 and greater. If you want to use + these types in earlier versions of Python, you should install the + typing-extensions_ package. + Large parts of the NumPy API have PEP-484-style type annotations. In addition, the following type aliases are available for users. @@ -13,6 +20,8 @@ inputs to ``np.array``" and ``typing.DtypeLike`` is "objects that can be used as inputs to ``np.dtype``". +.. _typing-extensions: https://pypi.org/project/typing-extensions/ + Differences from the runtime NumPy API -------------------------------------- diff --git a/numpy/typing/_array_like.py b/numpy/typing/_array_like.py index 54a612fb449c..b73585cf0ae5 100644 --- a/numpy/typing/_array_like.py +++ b/numpy/typing/_array_like.py @@ -1,5 +1,5 @@ import sys -from typing import Any, overload, Sequence, Tuple, Union +from typing import Any, overload, Sequence, TYPE_CHECKING, Union from numpy import ndarray from ._dtype_like import DtypeLike @@ -15,7 +15,7 @@ else: HAVE_PROTOCOL = True -if HAVE_PROTOCOL: +if TYPE_CHECKING or HAVE_PROTOCOL: class _SupportsArray(Protocol): @overload def __array__(self, __dtype: DtypeLike = ...) -> ndarray: ... From c88f5a232222b777a00e40648a6121428858df8d Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Sun, 14 Jun 2020 16:41:32 -0700 Subject: [PATCH 4/7] DOC: add release note about `np.typing` being available at runtime --- doc/release/upcoming_changes/16558.new_feature.rst | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/release/upcoming_changes/16558.new_feature.rst diff --git a/doc/release/upcoming_changes/16558.new_feature.rst b/doc/release/upcoming_changes/16558.new_feature.rst new file mode 100644 index 000000000000..9bd508e83a78 --- /dev/null +++ b/doc/release/upcoming_changes/16558.new_feature.rst @@ -0,0 +1,9 @@ +``numpy.typing`` is accessible at runtime +----------------------------------------- +The types in ``numpy.typing`` can now be imported at runtime. Code +like the following will now work: + +.. code:: python + + from numpy.typing import ArrayLike + x: ArrayLike = [1, 2, 3, 4] From c63f2333288772defcd84627986b035b6e7018ef Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 15 Jun 2020 20:56:57 -0700 Subject: [PATCH 5/7] DOC: clarify `ArrayLike` example in typing docs --- numpy/typing/__init__.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py index 1d377f8c46e3..f2000823fb61 100644 --- a/numpy/typing/__init__.py +++ b/numpy/typing/__init__.py @@ -39,10 +39,26 @@ .. code-block:: python - np.array(x**2 for x in range(10)) + >>> np.array(x**2 for x in range(10)) + array( at 0x10c004cd0>, dtype=object) -is valid NumPy code which will create an object array. The types will -complain about this usage however. +is valid NumPy code which will create a 0-dimensional object +array. Type checkers will complain about the above example when using +the NumPy types however. If you really intended to do the above, then +you can either use a ``# type: ignore`` comment: + +.. code-block:: python + + >>> np.array(x**2 for x in range(10)) # type: ignore + +or explicitly type the array like object as ``Any``: + +.. code-block:: python + + >>> from typing import Any + >>> array_like: Any = (x**2 for x in range(10)) + >>> np.array(array_like) + array( at 0x1192741d0>, dtype=object) ndarray ~~~~~~~ @@ -52,8 +68,8 @@ .. code-block:: python - x = np.array([1, 2]) - x.dtype = np.bool_ + x = np.array([1, 2]) + x.dtype = np.bool_ This sort of mutation is not allowed by the types. Users who want to write statically typed code should insted use the `numpy.ndarray.view` From 347a368cd937b73c74ec8f684dcbaacb233ec84a Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 15 Jun 2020 20:57:24 -0700 Subject: [PATCH 6/7] DOC: add note about supporting buffer protocols in `ArrayLike` --- numpy/typing/_array_like.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/numpy/typing/_array_like.py b/numpy/typing/_array_like.py index b73585cf0ae5..a4c526778f05 100644 --- a/numpy/typing/_array_like.py +++ b/numpy/typing/_array_like.py @@ -24,4 +24,9 @@ def __array__(self, dtype: DtypeLike = ...) -> ndarray: ... else: _SupportsArray = Any +# TODO: support buffer protocols once +# +# https://github.com/python/typing/issues/593 +# +# is resolved. ArrayLike = Union[bool, int, float, complex, _SupportsArray, Sequence] From d985e8ca2fca154d3770c842a2da1ba6dc3aaf1c Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 16 Jun 2020 21:32:29 -0700 Subject: [PATCH 7/7] DOC: add reference to Python issue about buffer protocols --- numpy/typing/_array_like.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/numpy/typing/_array_like.py b/numpy/typing/_array_like.py index a4c526778f05..76c0c839c567 100644 --- a/numpy/typing/_array_like.py +++ b/numpy/typing/_array_like.py @@ -26,7 +26,9 @@ def __array__(self, dtype: DtypeLike = ...) -> ndarray: ... # TODO: support buffer protocols once # -# https://github.com/python/typing/issues/593 +# https://bugs.python.org/issue27501 +# +# is resolved. See also the mypy issue: # -# is resolved. +# https://github.com/python/typing/issues/593 ArrayLike = Union[bool, int, float, complex, _SupportsArray, Sequence]