numpy · mattip · May 27, 2020 · Jan 31, 2020 · Feb 5, 2020 · Feb 6, 2020
diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt
@@ -51,3 +51,6 @@
 # Version 13 (NumPy 1.18) No change.
 # Version 13 (NumPy 1.19) No change.
 0x0000000d = 5b0e8bbded00b166125974fc71e80a33
+
+# Version 14 (NumPy 1.19) DType related API additions
+0x0000000e = 17a0f366e55ec05e5c5c149123478452
diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py
@@ -37,6 +37,7 @@
              join('multiarray', 'datetime_busdaycal.c'),
              join('multiarray', 'datetime_strings.c'),
              join('multiarray', 'descriptor.c'),
+             join('multiarray', 'dtypemeta.c'),
              join('multiarray', 'einsum.c.src'),
              join('multiarray', 'flagsobject.c'),
              join('multiarray', 'getset.c'),
@@ -309,11 +310,13 @@ def write_file(filename, data):
 
 # Those *Api classes instances know how to output strings for the generated code
 class TypeApi:
-    def __init__(self, name, index, ptr_cast, api_name):
+    def __init__(self, name, index, ptr_cast, api_name, internal_type=None):
         self.index = index
         self.name = name
         self.ptr_cast = ptr_cast
         self.api_name = api_name
+        # The type used internally, if None, same as exported (ptr_cast)
+        self.internal_type = internal_type
 
     def define_from_array_api_string(self):
         return "#define %s (*(%s *)%s[%d])" % (self.name,
@@ -325,9 +328,19 @@ def array_api_define(self):
         return "        (void *) &%s" % self.name
 
     def internal_define(self):
-        astr = """\
-extern NPY_NO_EXPORT PyTypeObject %(type)s;
-""" % {'type': self.name}
+        if self.internal_type is None:
+            return f"extern NPY_NO_EXPORT {self.ptr_cast} {self.name};\n"
+
+        # If we are here, we need to define a larger struct internally, which
+        # the type can be cast safely. But we want to normally use the original
+        # type, so name mangle:
+        mangled_name = f"{self.name}Full"
+        astr = (
+            # Create the mangled name:
+            f"extern NPY_NO_EXPORT {self.internal_type} {mangled_name};\n"
+            # And define the name as: (*(type *)(&mangled_name))
+            f"#define {self.name} (*({self.ptr_cast} *)(&{mangled_name}))\n"
+        )
         return astr
 
 class GlobalVarApi:

diff --git a/numpy/core/code_generators/generate_numpy_api.py b/numpy/core/code_generators/generate_numpy_api.py
@@ -201,7 +201,9 @@ def do_generate_api(targets, sources):
 
     for name, val in types_api.items():
         index = val[0]
-        multiarray_api_dict[name] = TypeApi(name, index, 'PyTypeObject', api_name)
+        internal_type =  None if len(val) == 1 else val[1]
+        multiarray_api_dict[name] = TypeApi(
+            name, index, 'PyTypeObject', api_name, internal_type)
 
     if len(multiarray_api_dict) != len(multiarray_api_index):
         keys_dict = set(multiarray_api_dict.keys())

diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py
@@ -30,7 +30,9 @@
 multiarray_types_api = {
     'PyBigArray_Type':                  (1,),
     'PyArray_Type':                     (2,),
-    'PyArrayDescr_Type':                (3,),
+    # Internally, PyArrayDescr_Type is a PyArray_DTypeMeta,
+    # the following also defines PyArrayDescr_TypeFull (Full appended)
+    'PyArrayDescr_Type':                (3, "PyArray_DTypeMeta"),
     'PyArrayFlags_Type':                (4,),
     'PyArrayIter_Type':                 (5,),
     'PyArrayMultiIter_Type':            (6,),

diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h
@@ -1810,6 +1810,77 @@ typedef struct {
 typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size,
                                        void *user_data);
 
+
+/*
+ * PyArray_DTypeMeta related definitions.
+ *
+ * As of now, this API is preliminary and will be extended as necessary.
+ */
+#if defined(NPY_INTERNAL_BUILD) && NPY_INTERNAL_BUILD
+    /*
+     * The Structures defined in this block are considered private API and
+     * may change without warni
6D40
ng!
+     */
+    /* TODO: Make this definition public in the API, as soon as its settled */
+    NPY_NO_EXPORT PyTypeObject PyArrayDTypeMeta_Type;
+
+    /*
+     * While NumPy DTypes would not need to be heap types the plan is to
+     * make DTypes available in Python at which point we will probably want
+     * them to be.
+     * Since we also wish to add fields to the DType class, this looks like
+     * a typical instance definition, but with PyHeapTypeObject instead of
+     * only the PyObject_HEAD.
+     * This must only be exposed very extremely careful consideration, since
+     * it is a fairly complex construct which may be better to allow
+     * refactoring of.
+     */
+    typedef struct _PyArray_DTypeMeta {
+        PyHeapTypeObject super;
+
+        /*
+         * Most DTypes will have a singleton default instance, for the
+         * parametric legacy DTypes (bytes, string, void, datetime) this
+         * may be a pointer to the *prototype* instance?
+         */
+        PyArray_Descr *singleton;
+        /*
+         * Is this DType created using the old API? This exists mainly to
+         * allow for assertions in paths specific to wrapping legacy types.
+         */
+        npy_bool legacy;
+        /* The values stored by a parametric datatype depend on its instance */
+        npy_bool parametric;
+        /* whether the DType can be instantiated (i.e. np.dtype cannot) */
+        npy_bool abstract;
+
+        /*
+         * The following fields replicate the most important dtype information.
+         * In the legacy implementation most of these are stored in the
+         * PyArray_Descr struct.
+         */
+        /* The type object of the scalar instances (may be NULL?) */
+        PyTypeObject *scalar_type;
+        /* kind for this type */
+        char kind;
+        /* unique-character representing this type */
+        char type;
+        /* flags describing data type */
+        char flags;
+        /* number representing this type */
+        int type_num;
+        /*
+         * Point to the original ArrFuncs.
+         * NOTE: We could make a copy to detect changes to `f`.
+         */
+        PyArray_ArrFuncs *f;
+    } PyArray_DTypeMeta;
+
+    #define NPY_DTYPE(descr) ((PyArray_DTypeMeta *)Py_TYPE(descr))
+
+#endif  /* NPY_INTERNAL_BUILD */
+
+
 /*
  * Use the keyword NPY_DEPRECATED_INCLUDES to ensure that the header files
  * npy_*_*_deprecated_api.h are only included from here and nowhere else.

diff --git a/numpy/core/setup.py b/numpy/core/setup.py
@@ -783,6 +783,7 @@ def get_mathlib_info(*args):
             join('src', 'multiarray', 'conversion_utils.h'),
             join('src', 'multiarray', 'ctors.h'),
             join('src', 'multiarray', 'descriptor.h'),
+            join('src', 'multiarray', 'dtypemeta.h'),
             join('src', 'multiarray', 'dragon4.h'),
             join('src', 'multiarray', 'getset.h'),
             join('src', 'multiarray', 'hashdescr.h'),
@@ -841,6 +842,7 @@ def get_mathlib_info(*args):
             join('src', 'multiarray', 'datetime_busday.c'),
             join('src', 'multiarray', 'datetime_busdaycal.c'),
             join('src', 'multiarray', 'descriptor.c'),
+            join('src', 'multiarray', 'dtypemeta.c'),
             join('src', 'multiarray', 'dragon4.c'),
             join('src', 'multiarray', 'dtype_transfer.c'),
             join('src', 'multiarray', 'einsum.c.src'),

diff --git a/numpy/core/setup_common.py b/numpy/core/setup_common.py
@@ -40,7 +40,8 @@
 # 0x0000000c - 1.14.x
 # 0x0000000c - 1.15.x
 # 0x0000000d - 1.16.x
-C_API_VERSION = 0x0000000d
+# 0x0000000e - 1.19.x
+C_API_VERSION = 0x0000000e
 
 class MismatchCAPIWarning(Warning):
     pass

diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src
@@ -20,6 +20,7 @@
 #include "npy_sort.h"
 #include "common.h"
 #include "ctors.h"
+#include "dtypemeta.h"
 #include "lowlevel_strided_loops.h"
 #include "usertypes.h"
 #include "_datetime.h"
@@ -4358,6 +4359,17 @@ set_typeinfo(PyObject *dict)
     PyArray_Descr *dtype;
     PyObject *cobj, *key;
 
+    /*
+     * Override the base class for all types, eventually all of this logic
+     * should be defined on the class and inherited to the scalar.
+     * (NPY_HALF is the largest builtin one.)
+     */
+    for (i = 0; i <= NPY_HALF; i++) {
+        if (dtypemeta_wrap_legacy_descriptor(_builtin_descrs[i]) < 0) {
+            return -1;
+        }
+    }
+
     /*
      * Add cast functions for the new types
      */

diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c
@@ -1744,7 +1744,7 @@ _convert_from_str(PyObject *obj, int align)
 NPY_NO_EXPORT PyArray_Descr *
 PyArray_DescrNew(PyArray_Descr *base)
 {
-    PyArray_Descr *newdescr = PyObject_New(PyArray_Descr, &PyArrayDescr_Type);
+    PyArray_Descr *newdescr = PyObject_New(PyArray_Descr, Py_TYPE(base));
 
     if (newdescr == NULL) {
         return NULL;
@@ -2261,9 +2261,16 @@ static PyGetSetDef arraydescr_getsets[] = {
 };
 
 static PyObject *
-arraydescr_new(PyTypeObject *NPY_UNUSED(subtype),
+arraydescr_new(PyTypeObject *subtype,
                 PyObject *args, PyObject *kwds)
 {
+    if (subtype != &PyArrayDescr_Type) {
+        /* The DTypeMeta class should prevent this from happening. */
+        PyErr_Format(PyExc_SystemError,
+                "'%S' must not inherit np.dtype.__new__().", subtype);
+        return NULL;
+    }
+
     PyObject *odescr, *metadata=NULL;
     PyArray_Descr *descr, *conv;
     npy_bool align = NPY_FALSE;
@@ -2334,6 +2341,7 @@ arraydescr_new(PyTypeObject *NPY_UNUSED(subtype),
     return (PyObject *)conv;
 }
 
+
 /*
  * Return a tuple of
  * (cleaned metadata dictionary, tuple with (str, num))
@@ -3456,21 +3464,34 @@ static PyMappingMethods descr_as_mapping = {
 
 /****************** End of Mapping Protocol ******************************/
 
-NPY_NO_EXPORT PyTypeObject PyArrayDescr_Type = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    .tp_name = "numpy.dtype",
-    .tp_basicsize = sizeof(PyArray_Descr),
-    /* methods */
-    .tp_dealloc = (destructor)arraydescr_dealloc,
-    .tp_repr = (reprfunc)arraydescr_repr,
-    .tp_as_number = &descr_as_number,
-    .tp_as_sequence = &descr_as_sequence,
-    .tp_as_mapping = &descr_as_mapping,
-    .tp_str = (reprfunc)arraydescr_str,
-    .tp_flags = Py_TPFLAGS_DEFAULT,
-    .tp_richcompare = (richcmpfunc)arraydescr_richcompare,
-    .tp_methods = arraydescr_methods,
-    .tp_members = arraydescr_members,
-    .tp_getset = arraydescr_getsets,
-    .tp_new = arraydescr_new,
+
+/*
+ * NOTE: Since this is a MetaClass, the name has Full appended here, the
+ *       correct name of the type is PyArrayDescr_Type.
+ */
+NPY_NO_EXPORT PyArray_DTypeMeta PyArrayDescr_TypeFull = {
+    {{
+        /* NULL represents `type`, this is set to DTypeMeta at import time */
+        PyVarObject_HEAD_INIT(NULL, 0)
+        .tp_name = "numpy.dtype",
+        .tp_basicsize = sizeof(PyArray_Descr),
+        .tp_dealloc = (destructor)arraydescr_dealloc,
+        .tp_repr = (reprfunc)arraydescr_repr,
+        .tp_as_number = &descr_as_number,
+        .tp_as_sequence = &descr_as_sequence,
+        .tp_as_mapping = &descr_as_mapping,
+        .tp_str = (reprfunc)arraydescr_str,
+        .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+        .tp_richcompare = (richcmpfunc)arraydescr_richcompare,
+        .tp_methods = arraydescr_methods,
+        .tp_members = arraydescr_members,
+        .tp_getset = arraydescr_getsets,
+        .tp_new = arraydescr_new,
+    },},
+    .type_num = -1,
+    .kind = '\0',
+    .abstract = 1,
+    .parametric = 0,
+    .singleton = 0,
+    .scalar_type = NULL,
 };