From fd0f3dd2723ed7effde52bf31a673c9128a0a28a Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Sun, 5 Jul 2020 15:32:09 -0700 Subject: [PATCH 1/4] ENH: make dtype generic over scalar type This allows representing dtype subclasses via constructs like `np.dtype[np.float64]`. --- numpy/__init__.pyi | 108 ++++++++++++++++++++++-- numpy/typing/__init__.py | 2 +- numpy/typing/_dtype_like.py | 30 ++++--- numpy/typing/tests/data/fail/dtype.py | 9 +- numpy/typing/tests/data/reveal/dtype.py | 24 ++++++ 5 files changed, 151 insertions(+), 22 deletions(-) create mode 100644 numpy/typing/tests/data/reveal/dtype.py diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 3d40682e7432..139f2a1bc580 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -15,6 +15,8 @@ from numpy.typing import ( _FloatLike, _ComplexLike, _NumberLike, + _SupportsDtype, + _VoidDtypeLike, ) from numpy.typing._callable import ( _BoolOp, @@ -527,16 +529,112 @@ where: Any who: Any _NdArraySubClass = TypeVar("_NdArraySubClass", bound=ndarray) +_DTypeScalar = TypeVar("_DTypeScalar", bound=generic) _ByteOrder = Literal["S", "<", ">", "=", "|", "L", "B", "N", "I"] -class dtype: +class dtype(Generic[_DTypeScalar]): names: Optional[Tuple[str, ...]] - def __init__( - self, - dtype: DtypeLike, + # Overload for subclass of generic + @overload + def __new__( + cls, + dtype: Type[_DTypeScalar], align: bool = ..., copy: bool = ..., - ) -> None: ... + ) -> dtype[_DTypeScalar]: ... + # Overloads for string aliases + @overload + def __new__( + cls, + dtype: Literal["float64", "f8", "f8", "float", "double", "float_", "d"], + align: bool = ..., + copy: bool = ..., + ) -> dtype[float64]: ... + @overload + def __new__( + cls, + dtype: Literal["float32", "f4", "f4", "single"], + align: bool = ..., + copy: bool = ..., + ) -> dtype[float32]: ... + @overload + def __new__( + cls, + dtype: Literal["int64", "i8", "i8"], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int64]: ... + @overload + def __new__( + cls, + dtype: Literal["int32", "i4", "i4"], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int32]: ... + # "int" resolves to int_, which is system dependent, and as of now + # untyped. Long-term we'll do something fancier here. + @overload + def __new__( + cls, + dtype: Literal["int"], + align: bool = ..., + copy: bool = ..., + ) -> dtype: ... + # Overloads for Python types. Order is important here. + @overload + def __new__( + cls, + dtype: Type[bool], + align: bool = ..., + copy: bool = ..., + ) -> dtype[bool_]: ... + # See the notes for "int" + @overload + def __new__( + cls, + dtype: Type[int], + align: bool = ..., + copy: bool = ..., + ) -> dtype[Any]: ... + @overload + def __new__( + cls, + dtype: Type[float], + align: bool = ..., + copy: bool = ..., + ) -> dtype[float64]: ... + # None is a special case + @overload + def __new__( + cls, + dtype: None, + align: bool = ..., + copy: bool = ..., + ) -> dtype[float64]: ... + # dtype of a dtype is the same dtype + @overload + def __new__( + cls, + dtype: dtype[_DTypeScalar], + align: bool = ..., + copy: bool = ..., + ) -> dtype[_DTypeScalar]: ... + # TODO: handle _SupportsDtype better + @overload + def __new__( + cls, + dtype: _SupportsDtype, + align: bool = ..., + copy: bool = ..., + ) -> dtype[Any]: ... + # Catchall overload + @overload + def __new__( + cls, + dtype: _VoidDtypeLike, + align: bool = ..., + copy: bool = ..., + ) -> dtype[void]: ... def __eq__(self, other: DtypeLike) -> bool: ... def __ne__(self, other: DtypeLike) -> bool: ... def __gt__(self, other: DtypeLike) -> bool: ... diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py index 987aa39aa1d5..dafabd95aefd 100644 --- a/numpy/typing/__init__.py +++ b/numpy/typing/__init__.py @@ -102,7 +102,7 @@ ) from ._array_like import _SupportsArray, ArrayLike from ._shape import _Shape, _ShapeLike -from ._dtype_like import DtypeLike +from ._dtype_like import _SupportsDtype, _VoidDtypeLike, DtypeLike from numpy._pytesttester import PytestTester test = PytestTester(__name__) diff --git a/numpy/typing/_dtype_like.py b/numpy/typing/_dtype_like.py index 7c1946a3e8b5..5bfd8ffdc0d6 100644 --- a/numpy/typing/_dtype_like.py +++ b/numpy/typing/_dtype_like.py @@ -38,18 +38,9 @@ class _SupportsDtype(Protocol): _DtypeDict = Any _SupportsDtype = Any -# Anything that can be coerced into numpy.dtype. -# Reference: https://docs.scipy.org/doc/numpy/reference/arrays.dtypes.html -DtypeLike = Union[ - dtype, - # default data type (float64) - None, - # array-scalar types and generic types - type, # TODO: enumerate these when we add type hints for numpy scalars - # anything with a dtype attribute - _SupportsDtype, - # character codes, type strings or comma-separated fields, e.g., 'float64' - str, + +# Would create a dtype[np.void] +_VoidDtypeLike = Union[ # (flexible_dtype, itemsize) Tuple[_DtypeLikeNested, int], # (fixed_dtype, shape) @@ -67,6 +58,21 @@ class _SupportsDtype(Protocol): Tuple[_DtypeLikeNested, _DtypeLikeNested], ] +# Anything that can be coerced into numpy.dtype. +# Reference: https://docs.scipy.org/doc/numpy/reference/arrays.dtypes.html +DtypeLike = Union[ + dtype, + # default data type (float64) + None, + # array-scalar types and generic types + type, # TODO: enumerate these when we add type hints for numpy scalars + # anything with a dtype attribute + _SupportsDtype, + # character codes, type strings or comma-separated fields, e.g., 'float64' + str, + _VoidDtypeLike, +] + # NOTE: while it is possible to provide the dtype as a dict of # dtype-like objects (e.g. `{'field1': ..., 'field2': ..., ...}`), # this syntax is officially discourged and diff --git a/numpy/typing/tests/data/fail/dtype.py b/numpy/typing/tests/data/fail/dtype.py index 3dc027daf243..7d4783d8f651 100644 --- a/numpy/typing/tests/data/fail/dtype.py +++ b/numpy/typing/tests/data/fail/dtype.py @@ -1,15 +1,16 @@ import numpy as np - class Test: not_dtype = float -np.dtype(Test()) # E: Argument 1 to "dtype" has incompatible type +np.dtype(Test()) # E: No overload variant of "dtype" matches -np.dtype( - { # E: Argument 1 to "dtype" has incompatible type +np.dtype( # E: No overload variant of "dtype" matches + { "field1": (float, 1), "field2": (int, 3), } ) + +np.dtype[np.float64](np.int64) # E: Argument 1 to "dtype" has incompatible type diff --git a/numpy/typing/tests/data/reveal/dtype.py b/numpy/typing/tests/data/reveal/dtype.py new file mode 100644 index 000000000000..aca7e8a5e983 --- /dev/null +++ b/numpy/typing/tests/data/reveal/dtype.py @@ -0,0 +1,24 @@ +import numpy as np + +reveal_type(np.dtype(np.float64)) # E: numpy.dtype[numpy.float64*] +reveal_type(np.dtype(np.int64)) # E: numpy.dtype[numpy.int64*] + +# String aliases +reveal_type(np.dtype("float64")) # E: numpy.dtype[numpy.float64] +reveal_type(np.dtype("float32")) # E: numpy.dtype[numpy.float32] +reveal_type(np.dtype("int64")) # E: numpy.dtype[numpy.int64] +reveal_type(np.dtype("int32")) # E: numpy.dtype[numpy.int32] + +# Python types +reveal_type(np.dtype(float)) # E: numpy.dtype[numpy.float64] +reveal_type(np.dtype(int)) # E: numpy.dtype +reveal_type(np.dtype(bool)) # E: numpy.dtype[numpy.bool_] + +# Special case for None +reveal_type(np.dtype(None)) # E: numpy.dtype[numpy.float64] + +# Dtypes of dtypes +reveal_type(np.dtype(np.dtype(np.float64))) # E: numpy.dtype[numpy.float64*] + +# Void +reveal_type(np.dtype(("U", 10))) # E: numpy.dtype[numpy.void] From 02688c220591250082d4ce109eb51421d8412099 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 6 Oct 2020 22:20:47 -0700 Subject: [PATCH 2/4] MAINT: add more dtype __new__ overloads for missing scalar types --- numpy/__init__.pyi | 282 ++++++++++++++++++++++-- numpy/typing/tests/data/reveal/dtype.py | 9 + 2 files changed, 270 insertions(+), 21 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 139f2a1bc580..f4caaab7ccdc 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -542,75 +542,307 @@ class dtype(Generic[_DTypeScalar]): align: bool = ..., copy: bool = ..., ) -> dtype[_DTypeScalar]: ... - # Overloads for string aliases + # Overloads for string aliases, Python types, and some assorted + # other special cases. Order is sometimes important because of the + # subtype relationships + # + # bool < int < float < complex + # + # so we have to make sure the overloads for the narrowest type is + # first. @overload def __new__( cls, - dtype: Literal["float64", "f8", "f8", "float", "double", "float_", "d"], + dtype: Union[ + Type[bool], + Literal[ + "?", + "=?", + "?", + "bool", + "bool_", + ], + ], align: bool = ..., copy: bool = ..., - ) -> dtype[float64]: ... + ) -> dtype[bool_]: ... @overload def __new__( cls, - dtype: Literal["float32", "f4", "f4", "single"], + dtype: Literal[ + "uint8", + "u1", + "=u1", + "u1", + "B", + "=B", + "B", + ], align: bool = ..., copy: bool = ..., - ) -> dtype[float32]: ... + ) -> dtype[uint8]: ... @overload def __new__( cls, - dtype: Literal["int64", "i8", "i8"], + dtype: Literal[ + "uint16", + "u2", + "=u2", + "u2", + "h", + "=h", + "h", + ], align: bool = ..., copy: bool = ..., - ) -> dtype[int64]: ... + ) -> dtype[uint16]: ... @overload def __new__( cls, - dtype: Literal["int32", "i4", "i4"], + dtype: Literal[ + "uint32", + "u4", + "=u4", + "u4", + "I", + "=I", + "I", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[uint32]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "uint64", + "u8", + "=u8", + "u8", + "L", + "=L", + "L", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[uint64]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "int8", + "i1", + "=i1", + "i1", + "b", + "=b", + "b", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int8]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "int16", + "i2", + "=i2", + "i2", + "h", + "=h", + "h", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int16]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "int32", + "i4", + "=i4", + "i4", + "i", + "=i", + "i", + ], align: bool = ..., copy: bool = ..., ) -> dtype[int32]: ... - # "int" resolves to int_, which is system dependent, and as of now - # untyped. Long-term we'll do something fancier here. @overload def __new__( cls, - dtype: Literal["int"], + dtype: Literal[ + "int64", + "i8", + "=i8", + "i8", + "l", + "=l", + "l", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int64]: ... + # "int"/int resolve to int_, which is system dependent and as of + # now untyped. Long-term we'll do something fancier here. + @overload + def __new__( + cls, + dtype: Union[Type[int], Literal["int"]], align: bool = ..., copy: bool = ..., ) -> dtype: ... - # Overloads for Python types. Order is important here. @overload def __new__( cls, - dtype: Type[bool], + dtype: Literal[ + "float16", + "f4", + "=f4", + "f4", + "e", + "=e", + "e", + "half", + ], align: bool = ..., copy: bool = ..., - ) -> dtype[bool_]: ... - # See the notes for "int" + ) -> dtype[float16]: ... @overload def __new__( cls, - dtype: Type[int], + dtype: Literal[ + "float32", + "f4", + "=f4", + "f4", + "f", + "=f", + "f", + "single", + ], align: bool = ..., copy: bool = ..., - ) -> dtype[Any]: ... + ) -> dtype[float32]: ... @overload def __new__( cls, - dtype: Type[float], + dtype: Union[ + None, + Type[float], + Literal[ + "float64", + "f8", + "=f8", + "f8", + "d", + "d", + "float", + "double", + "float_", + ], + ], align: bool = ..., copy: bool = ..., ) -> dtype[float64]: ... - # None is a special case @overload def __new__( cls, - dtype: None, + dtype: Literal[ + "complex64", + "c8", + "=c8", + "c8", + "F", + "=F", + "F", + ], align: bool = ..., copy: bool = ..., - ) -> dtype[float64]: ... + ) -> dtype[complex128]: ... + @overload + def __new__( + cls, + dtype: Union[ + Type[complex], + Literal[ + "complex128", + "c16", + "=c16", + "c16", + "D", + "=D", + "D", + ], + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[complex128]: ... + @overload + def __new__( + cls, + dtype: Union[ + Type[bytes], + Literal[ + "S", + "=S", + "S", + "bytes", + "bytes_", + ], + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[bytes_]: ... + @overload + def __new__( + cls, + dtype: Union[ + Type[str], + Literal[ + "U", + "=U", + # U intentionally not included; they are not + # the same dtype and which one dtype("U") translates + # to is platform-dependent. + "str", + "str_", + ], + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[str_]: ... # dtype of a dtype is the same dtype @overload def __new__( @@ -627,6 +859,14 @@ class dtype(Generic[_DTypeScalar]): align: bool = ..., copy: bool = ..., ) -> dtype[Any]: ... + # Handle strings that can't be expressed as literals; i.e. s1, s2, ... + @overload + def __new__( + cls, + dtype: str, + align: bool = ..., + copy: bool = ..., + ) -> dtype[Any]: ... # Catchall overload @overload def __new__( diff --git a/numpy/typing/tests/data/reveal/dtype.py b/numpy/typing/tests/data/reveal/dtype.py index aca7e8a5e983..e0802299e973 100644 --- a/numpy/typing/tests/data/reveal/dtype.py +++ b/numpy/typing/tests/data/reveal/dtype.py @@ -8,11 +8,17 @@ reveal_type(np.dtype("float32")) # E: numpy.dtype[numpy.float32] reveal_type(np.dtype("int64")) # E: numpy.dtype[numpy.int64] reveal_type(np.dtype("int32")) # E: numpy.dtype[numpy.int32] +reveal_type(np.dtype("bool")) # E: numpy.dtype[numpy.bool_] +reveal_type(np.dtype("bytes")) # E: numpy.dtype[numpy.bytes_] +reveal_type(np.dtype("str")) # E: numpy.dtype[numpy.str_] # Python types +reveal_type(np.dtype(complex)) # E: numpy.dtype[numpy.complex128] reveal_type(np.dtype(float)) # E: numpy.dtype[numpy.float64] reveal_type(np.dtype(int)) # E: numpy.dtype reveal_type(np.dtype(bool)) # E: numpy.dtype[numpy.bool_] +reveal_type(np.dtype(str)) # E: numpy.dtype[numpy.str_] +reveal_type(np.dtype(bytes)) # E: numpy.dtype[numpy.bytes_] # Special case for None reveal_type(np.dtype(None)) # E: numpy.dtype[numpy.float64] @@ -20,5 +26,8 @@ # Dtypes of dtypes reveal_type(np.dtype(np.dtype(np.float64))) # E: numpy.dtype[numpy.float64*] +# Parameterized dtypes +reveal_type(np.dtype("S8")) # E: numpy.dtype + # Void reveal_type(np.dtype(("U", 10))) # E: numpy.dtype[numpy.void] From 12e3e1a33ebbef9f3b263b258e3eabab6de5f6a5 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 7 Oct 2020 21:03:30 -0700 Subject: [PATCH 3/4] MAINT: fix complex64 overload; add str0/bytes to literals --- numpy/__init__.pyi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index f4caaab7ccdc..c51141d5de85 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -787,7 +787,7 @@ class dtype(Generic[_DTypeScalar]): ], align: bool = ..., copy: bool = ..., - ) -> dtype[complex128]: ... + ) -> dtype[complex64]: ... @overload def __new__( cls, @@ -820,6 +820,7 @@ class dtype(Generic[_DTypeScalar]): ">S", "bytes", "bytes_", + "bytes0", ], ], align: bool = ..., @@ -838,6 +839,7 @@ class dtype(Generic[_DTypeScalar]): # to is platform-dependent. "str", "str_", + "str0", ], ], align: bool = ..., From b81ab444c0e56011e96c8895a19e18906ab4e731 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Sun, 11 Oct 2020 14:30:32 -0700 Subject: [PATCH 4/4] MAINT: remove character code literals for integer types The various character codes are for C types, which are all platform-dependent. --- numpy/__init__.pyi | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index c51141d5de85..be4f55a32116 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -576,10 +576,6 @@ class dtype(Generic[_DTypeScalar]): "=u1", "u1", - "B", - "=B", - "B", ], align: bool = ..., copy: bool = ..., @@ -593,10 +589,6 @@ class dtype(Generic[_DTypeScalar]): "=u2", "u2", - "h", - "=h", - "h", ], align: bool = ..., copy: bool = ..., @@ -610,10 +602,6 @@ class dtype(Generic[_DTypeScalar]): "=u4", "u4", - "I", - "=I", - "I", ], align: bool = ..., copy: bool = ..., @@ -627,10 +615,6 @@ class dtype(Generic[_DTypeScalar]): "=u8", "u8", - "L", - "=L", - "L", ], align: bool = ..., copy: bool = ..., @@ -644,10 +628,6 @@ class dtype(Generic[_DTypeScalar]): "=i1", "i1", - "b", - "=b", - "b", ], align: bool = ..., copy: bool = ..., @@ -661,10 +641,6 @@ class dtype(Generic[_DTypeScalar]): "=i2", "i2", - "h", - "=h", - "h", ], align: bool = ..., copy: bool = ..., @@ -678,10 +654,6 @@ class dtype(Generic[_DTypeScalar]): "=i4", "i4", - "i", - "=i", - "i", ], align: bool = ..., copy: bool = ..., @@ -695,10 +667,6 @@ class dtype(Generic[_DTypeScalar]): "=i8", "i8", - "l", - "=l", - "l", ], align: bool = ..., copy: bool = ...,