8000 Merge pull request #15508 from seberg/dtypemeta-new · numpy/numpy@36e0171 · GitHub
[go: up one dir, main page]
Skip to content

Commit 36e0171

Browse files
mattipmattip
authored
Merge pull request #15508 from seberg/dtypemeta-new
API: Create Preliminary DTypeMeta class and np.dtype subclasses
2 parents 74ad82c + 2ea745b commit 36e0171

File tree

14 files changed

+481
-30
lines changed

14 files changed

+481
-30
lines changed

‎numpy/core/code_generators/cversions.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,6 @@
5252
# Version 13 (NumPy 1.19) No change.
5353
# Version 13 (NumPy 1.20) No change.
5454
0x0000000d = 5b0e8bbded00b166125974fc71e80a33
55+
56+
# Version 14 (NumPy 1.19) DType related API additions
57+
0x0000000e = 17a0f366e55ec05e5c5c149123478452

‎numpy/core/code_generators/genapi.py

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
join('multiarray', 'datetime_busdaycal.c'),
3838
join('multiarray', 'datetime_strings.c'),
3939
join('multiarray', 'descriptor.c'),
40+
join('multiarray', 'dtypemeta.c'),
4041
join('multiarray', 'einsum.c.src'),
4142
join('multiarray', 'flagsobject.c'),
4243
join('multiarray', 'getset.c'),
@@ -309,11 +310,13 @@ def write_file(filename, data):
309310

310311
# Those *Api classes instances know how to output strings for the generated code
311312
class TypeApi:
312-
def __init__(self, name, index, ptr_cast, api_name):
313+
def __init__(self, name, index, ptr_cast, api_name, internal_type=None):
313314
self.index = index
314315
self.name = name
315316
self.ptr_cast = ptr_cast
316317
self.api_name = api_name
318+
# The type used internally, if None, same as exported (ptr_cast)
319+
self.internal_type = internal_type
317320

318321
def define_from_array_api_string(self):
319322
return "#define %s (*(%s *)%s[%d])" % (self.name,
@@ -325,9 +328,19 @@ def array_api_define(self):
325328
return " (void *) &%s" % self.name
326329

327330
def internal_define(self):
328-
astr = """\
329-
extern NPY_NO_EXPORT PyTypeObject %(type)s;
330-
""" % {'type': self.name}
331+
if self.internal_type is None:
332+
return f"extern NPY_NO_EXPORT {self.ptr_cast} {self.name};\n"
333+
334+
# If we are here, we need to define a larger struct internally, which
335+
# the type can be cast safely. But we want to normally use the original
336+
# type, so name mangle:
337+
mangled_name = f"{self.name}Full"
338+
astr = (
339+
# Create the mangled name:
340+
f"extern NPY_NO_EXPORT {self.internal_type} {mangled_name};\n"
341+
# And define the name as: (*(type *)(&mangled_name))
342+
f"#define {self.name} (*({self.ptr_cast} *)(&{mangled_name}))\n"
343+
)
331344
return astr
332345

333346
class GlobalVarApi:

‎numpy/core/code_generators/generate_numpy_api.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,9 @@ def do_generate_api(targets, sources):
201201

202202
for name, val in types_api.items():
203203
index = val[0]
204-
multiarray_api_dict[name] = TypeApi(name, index, 'PyTypeObject', api_name)
204+
internal_type = None if len(val) == 1 else val[1]
205+
multiarray_api_dict[name] = TypeApi(
206+
name, index, 'PyTypeObject', api_name, internal_type)
205207

206208
if len(multiarray_api_dict) != len(multiarray_api_index):
207209
keys_dict = set(multiarray_api_dict.keys())

‎numpy/core/code_generators/numpy_api.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@
3030
multiarray_types_api = {
3131
'PyBigArray_Type': (1,),
3232
'PyArray_Type': (2,),
33-
'PyArrayDescr_Type': (3,),
33+
# Internally, PyArrayDescr_Type is a PyArray_DTypeMeta,
34+
# the following also defines PyArrayDescr_TypeFull (Full appended)
35+
'PyArrayDescr_Type': (3, "PyArray_DTypeMeta"),
3436
'PyArrayFlags_Type': (4,),
3537
'PyArrayIter_Type': (5,),
3638
'PyArrayMultiIter_Type': (6,),

‎numpy/core/include/numpy/ndarraytypes.h

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1809,6 +1809,77 @@ typedef struct {
18091809
typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size,
18101810
void *user_data);
18111811

1812+
1813+
/*
1814+
* PyArray_DTypeMeta related definitions.
1815+
*
1816+
* As of now, this API is preliminary and will be extended as necessary.
1817+
*/
1818+
#if defined(NPY_INTERNAL_BUILD) && NPY_INTERNAL_BUILD
1819+
/*
1820+
* The Structures defined in this block are considered private API and
1821+
* may change without warning!
1822+
*/
1823+
/* TODO: Make this definition public in the API, as soon as its settled */
1824+
NPY_NO_EXPORT PyTypeObject PyArrayDTypeMeta_Type;
1825+
1826+
/*
1827+
* While NumPy DTypes would not need to be heap types the plan is to
1828+
* make DTypes available in Python at which point we will probably want
1829+
* them to be.
1830+
* Since we also wish to add fields to the DType class, this looks like
1831+
* a typical instance definition, but with PyHeapTypeObject instead of
1832+
* only the PyObject_HEAD.
1833+
* This must only be exposed very extremely careful consideration, since
1834+
* it is a fairly complex construct which may be better to allow
1835+
* refactoring of.
1836+
*/
1837+
typedef struct _PyArray_DTypeMeta {
1838+
PyHeapTypeObject super;
1839+
1840+
/*
1841+
* Most DTypes will have a singleton default instance, for the
1842+
* parametric legacy DTypes (bytes, string, void, datetime) this
1843+
* may be a pointer to the *prototype* instance?
1844+
*/
1845+
PyArray_Descr *singleton;
1846+
/*
1847+
* Is this DType created using the old API? This exists mainly to
1848+
* allow for assertions in paths specific to wrapping legacy types.
1849+
*/
1850+
npy_bool legacy;
1851+
/* The values stored by a parametric datatype depend on its instance */
1852+
npy_bool parametric;
1853+
/* whether the DType can be instantiated (i.e. np.dtype cannot) */
1854+
npy_bool abstract;
1855+
1856+
/*
1857+
* The following fields replicate the most important dtype information.
1858+
* In the legacy implementation most of these are stored in the
1859+
* PyArray_Descr struct.
1860+
*/
1861+
/* The type object of the scalar instances (may be NULL?) */
1862+
PyTypeObject *scalar_type;
1863+
/* kind for this type */
1864+
char kind;
1865+
/* unique-character representing this type */
1866+
char type;
1867+
/* flags describing data type */
1868+
char flags;
1869+
/* number representing this type */
1870+
int type_num;
1871+
/*
1872+
* Point to the original ArrFuncs.
1873+
* NOTE: We could make a copy to detect changes to `f`.
1874+
*/
1875+
PyArray_ArrFuncs *f;
1876+
} PyArray_DTypeMeta;
1877+
1878+
#define NPY_DTYPE(descr) ((PyArray_DTypeMeta *)Py_TYPE(descr))
1879+
1880+
#endif /* NPY_INTERNAL_BUILD */
1881+
1882+
18121883
/*
18131884
* Use the keyword NPY_DEPRECATED_INCLUDES to ensure that the header files
18141885
* npy_*_*_deprecated_api.h are only included from here and nowhere else.

‎numpy/core/setup.py

Lines changed: 2 additions & 0 deletions
784
Original file line numberDiff line numberDiff line change
@@ -784,6 +784,7 @@ def get_mathlib_info(*args):
784
join('src', 'multiarray', 'conversion_utils.h'),
785785
join('src', 'multiarray', 'ctors.h'),
786786
join('src', 'multiarray', 'descriptor.h'),
787+
join('src', 'multiarray', 'dtypemeta.h'),
787788
join('src', 'multiarray', 'dragon4.h'),
788789
join('src', 'multiarray', 'getset.h'),
789790
join('src', 'multiarray', 'hashdescr.h'),
@@ -842,6 +843,7 @@ def get_mathlib_info(*args):
842843
join('src', 'multiarray', 'datetime_busday.c'),
843844
join('src', 'multiarray', 'datetime_busdaycal.c'),
844845
join('src', 'multiarray', 'descriptor.c'),
846+
join('src', 'multiarray', 'dtypemeta.c'),
845847
join('src', 'multiarray', 'dragon4.c'),
846848
join('src', 'multiarray', 'dtype_transfer.c'),
847849
join('src', 'multiarray', 'einsum.c.src'),

‎numpy/core/setup_common.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@
4040
# 0x0000000c - 1.14.x
4141
# 0x0000000c - 1.15.x
4242
# 0x0000000d - 1.16.x
43-
C_API_VERSION = 0x0000000d
43+
# 0x0000000e - 1.19.x
44+
C_API_VERSION = 0x0000000e
4445

4546
class MismatchCAPIWarning(Warning):
4647
pass

‎numpy/core/src/multiarray/arraytypes.c.src

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "npy_sort.h"
2121
#include "common.h"
2222
#include "ctors.h"
23+
#include "dtypemeta.h"
2324
#include "lowlevel_strided_loops.h"
2425
#include "usertypes.h"
2526
#include "_datetime.h"
@@ -4366,6 +4367,17 @@ set_typeinfo(PyObject *dict)
43664367
PyArray_Descr *dtype;
43674368
PyObject *cobj, *key;
43684369

4370+
/*
4371+
* Override the base class for all types, eventually all of this logic
4372+
* should be defined on the class and inherited to the scalar.
4373+
* (NPY_HALF is the largest builtin one.)
4374+
*/
4375+
for (i = 0; i <= NPY_HALF; i++) {
4376+
if (dtypemeta_wrap_legacy_descriptor(_builtin_descrs[i]) < 0) {
4377+
return -1;
4378+
}
4379+
}
4380+
43694381
/*
43704382
* Add cast functions for the new types
43714383
*/

‎numpy/core/src/multiarray/descriptor.c

Lines changed: 40 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1744,7 +1744,7 @@ _convert_from_str(PyObject *obj, int align)
17441744
NPY_NO_EXPORT PyArray_Descr *
17451745
PyArray_DescrNew(PyArray_Descr *base)
17461746
{
1747-
PyArray_Descr *newdescr = PyObject_New(PyArray_Descr, &PyArrayDescr_Type);
1747+
PyArray_Descr *newdescr = PyObject_New(PyArray_Descr, Py_TYPE(base));
17481748

17491749
if (newdescr == NULL) {
17501750
return NULL;
@@ -2261,9 +2261,16 @@ static PyGetSetDef arraydescr_getsets[] = {
22612261
};
22622262

22632263
static PyObject *
2264-
arraydescr_new(PyTypeObject *NPY_UNUSED(subtype),
2264+
arraydescr_new(PyTypeObject *subtype,
22652265
PyObject *args, PyObject *kwds)
22662266
{
2267+
if (subtype != &PyArrayDescr_Type) {
2268+
/* The DTypeMeta class should prevent this from happening. */
2269+
PyErr_Format(PyExc_SystemError,
2270+
"'%S' must not inherit np.dtype.__new__().", subtype);
2271+
return NULL;
2272+
}
2273+
22672274
PyObject *odescr, *metadata=NULL;
22682275
PyArray_Descr *descr, *conv;
22692276
npy_bool align = NPY_FALSE;
@@ -2334,6 +2341,7 @@ arraydescr_new(PyTypeObject *NPY_UNUSED(subtype),
23342341
return (PyObject *)conv;
23352342
}
23362343

2344+
23372345
/*
23382346
* Return a tuple of
23392347
* (cleaned metadata dictionary, tuple with (str, num))
@@ -3456,21 +3464,34 @@ static PyMappingMethods descr_as_mapping = {
34563464

34573465
/****************** End of Mapping Protocol ******************************/
34583466

3459-
NPY_NO_EXPORT PyTypeObject PyArrayDescr_Type = {
3460-
PyVarObject_HEAD_INIT(NULL, 0)
3461-
.tp_name = "numpy.dtype",
3462-
.tp_basicsize = sizeof(PyArray_Descr),
3463-
/* methods */
3464-
.tp_dealloc = (destructor)arraydescr_dealloc,
3465-
.tp_repr = (reprfunc)arraydescr_repr,
3466-
.tp_as_number = &descr_as_number,
3467-
.tp_as_sequence = &descr_as_sequence,
3468-
.tp_as_mapping = &descr_as_mapping,
3469-
.tp_str = (reprfunc)arraydescr_str,
3470-
.tp_flags = Py_TPFLAGS_DEFAULT,
3471-
.tp_richcompare = (richcmpfunc)arraydescr_richcompare,
3472-
.tp_methods = arraydescr_methods,
3473-
.tp_members = arraydescr_members,
3474-
.tp_getset = arraydescr_getsets,
3475-
.tp_new = arraydescr_new,
3467+
3468+
/*
3469+
* NOTE: Since this is a MetaClass, the name has Full appended here, the
3470+
* correct name of the type is PyArrayDescr_Type.
3471+
*/
3472+
NPY_NO_EXPORT PyArray_DTypeMeta PyArrayDescr_TypeFull = {
3473+
{{
3474+
/* NULL represents `type`, this is set to DTypeMeta at import time */
3475+
PyVarObject_HEAD_INIT(NULL, 0)
3476+
.tp_name = "numpy.dtype",
3477+
.tp_basicsize = sizeof(PyArray_Descr),
3478+
.tp_dealloc = (destructor)arraydescr_dealloc,
3479+
.tp_repr = (reprfunc)arraydescr_repr,
3480+
.tp_as_number = &descr_as_number,
3481+
.tp_as_sequence = &descr_as_sequence,
3482+
.tp_as_mapping = &descr_as_mapping,
3483+
.tp_str = (reprfunc)arraydescr_str,
3484+
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
3485+
.tp_richcompare = (richcmpfunc)arraydescr_richcompare,
3486+
.tp_methods = arraydescr_methods,
3487+
.tp_members = arraydescr_members,
3488+
.tp_getset = arraydescr_getsets,
3489+
.tp_new = arraydescr_new,
3490+
},},
3491+
.type_num = -1,
3492+
.kind = '\0',
3493+
.abstract = 1,
3494+
.parametric = 0,
3495+
.singleton = 0,
3496+
.scalar_type = NULL,
34763497
};

0 commit comments

Comments
 (0)
0