numpy · charris · Nov 26, 2020 · Jul 23, 2020 · Nov 7, 2020 · Nov 12, 2020
diff --git a/doc/release/upcoming_changes/16938.c_api.rst b/doc/release/upcoming_changes/16938.c_api.rst
@@ -0,0 +1,19 @@
+Size of ``np.ndarray`` and ``np.void_`` changed
+-----------------------------------------------
+The size of the ``PyArrayObject`` and ``PyVoidScalarObject``
+structures have changed.  The following header definition has been
+removed::
+
+    #define NPY_SIZEOF_PYARRAYOBJECT (sizeof(PyArrayObject_fields))
+
+since the size must not be considered a compile time constant: it will
+change for different runtime versions of NumPy.
+
+The most likely relevant use are potential subclasses written in C which
+will have to be recompiled and should be updated.  Please see the
+documentation for :c:type:`PyArrayObject` for more details and contact
+the NumPy developers if you are affected by this change.
+
+NumPy will attempt to give a graceful error but a program expecting a
+fixed structure size may have undefined behaviour and likely crash.
+
diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst
@@ -79,6 +79,8 @@ PyArray_Type and PyArrayObject
    of :c:type:`NPY_AO` (deprecated) which is defined to be equivalent to
    :c:type:`PyArrayObject`. Direct access to the struct fields are
    deprecated. Use the ``PyArray_*(arr)`` form instead.
+   As of NumPy 1.20, the size of this struct is not considered part of
+   the NumPy ABI (see note at the end of the member list).
 
    .. code-block:: c
 
@@ -92,6 +94,7 @@ PyArray_Type and PyArrayObject
           PyArray_Descr *descr;
           int flags;
           PyObject *weakreflist;
+          /* version dependend private members */
       } PyArrayObject;
 
    .. c:macro:: PyObject_HEAD
@@ -173,6 +176,27 @@ PyArray_Type and PyArrayObject
        This member allows array objects to have weak references (using the
        weakref module).
 
+   .. note::
+
+      Further members are considered private and version dependend. If the size
+      of the struct is important for your code, special care must be taken.
+      A possible use-case when this is relevant is subclassing in C.
+      If your code relies on ``sizeof(PyArrayObject)`` to be constant,
+      you must add the following check at import time:
+
+      .. code-block:: c
+
+         if (sizeof(PyArrayObject) < PyArray_Type.tp_basicsize) {
+             PyErr_SetString(PyExc_ImportError,
+                "Binary incompatibility with NumPy, must recompile/update X.");
+             return NULL;
+         }
+
+      To ensure that your code does not have to be compiled for a specific
+      NumPy version, you may add a constant, leaving room for changes in NumPy.
+      A solution guaranteed to be compatible with any future NumPy version
+      requires the use of a runtime calculate offset and allocation size.
+
 
 PyArrayDescr_Type and PyArray_Descr
 -----------------------------------

diff --git a/numpy/core/include/numpy/arrayscalars.h b/numpy/core/include/numpy/arrayscalars.h
@@ -149,6 +149,7 @@ typedef struct {
         PyArray_Descr *descr;
         int flags;
         PyObject *base;
+        void *_buffer_info;  /* private buffer info, tagged to allow warning */
 } PyVoidScalarObject;
 
 /* Macros

diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h
@@ -701,6 +701,7 @@ typedef struct tagPyArrayObject_fields {
     int flags;
     /* For weak references */
     PyObject *weakreflist;
+    void *_buffer_info;  /* private buffer info, tagged to allow warning */
 } PyArrayObject_fields;
 
 /*
@@ -720,7 +721,18 @@ typedef struct tagPyArrayObject {
 } PyArrayObject;
 #endif
 
-#define NPY_SIZEOF_PYARRAYOBJECT (sizeof(PyArrayObject_fields))
+/*
+ * Removed 2020-Nov-25, NumPy 1.20
+ * #define NPY_SIZEOF_PYARRAYOBJECT (sizeof(PyArrayObject_fields))
+ *
+ * The above macro was removed as it gave a false sense of a stable ABI
+ * with respect to the structures size.  If you require a runtime constant,
+ * you can use `PyArray_Type.tp_basicsize` instead.  Otherwise, please
+ * see the PyArrayObject documentation or ask the NumPy developers for
+ * information on how to correctly replace the macro in a way that is
+ * compatible with multiple NumPy versions.
+ */
+
 
 /* Array Flags Object */
 typedef struct PyArrayFlagsObject {

diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src
@@ -36,6 +36,7 @@ IsPythonScalar(PyObject * dummy, PyObject *args)
 
 #include "npy_pycompat.h"
 
+
 /** Function to test calling via ctypes */
 EXPORT(void*) forward_pointer(void *x)
 {
@@ -684,6 +685,39 @@ create_custom_field_dtype(PyObject *NPY_UNUSED(mod), PyObject *args)
 }
 
 
+PyObject *
+corrupt_or_fix_bufferinfo(PyObject *dummy, PyObject *obj)
+{
+    void **buffer_info_ptr;
+    if (PyArray_Check(obj)) {
+        buffer_info_ptr = &((PyArrayObject_fields *)obj)->_buffer_info;
+    }
+    else if (PyArray_IsScalar(obj, Void)) {
+        buffer_info_ptr = &((PyVoidScalarObject *)obj)->_buffer_info;
+    }
+    else {
+        PyErr_SetString(PyExc_TypeError,
+                "argument must be an array or void scalar");
+        return NULL;
+    }
+    if (*buffer_info_ptr == NULL) {
+        /* set to an invalid value (as a subclass might accidentally) */
+        *buffer_info_ptr = obj;
+        assert(((uintptr_t)obj & 7) == 0);
+    }
+    else if (*buffer_info_ptr == obj) {
+        /* Reset to a NULL (good value) */
+        *buffer_info_ptr = NULL;
+    }
+    else {
+        PyErr_SetString(PyExc_TypeError,
+                "buffer was already exported, this test doesn't support that");
+        return NULL;
+    }
+    Py_RETURN_NONE;
+}
+
+
 /* check no elison for avoided increfs */
 static PyObject *
 incref_elide(PyObject *dummy, PyObject *args)
@@ -2158,6 +2192,9 @@ static PyMethodDef Multiarray_TestsMethods[] = {
     {"create_custom_field_dtype",
         create_custom_field_dtype,
         METH_VARARGS, NULL},
+    {"corrupt_or_fix_bufferinfo",
+        corrupt_or_fix_bufferinfo,
+        METH_O, NULL},
     {"incref_elide",
         incref_elide,
         METH_VARARGS, NULL},

diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c
@@ -434,7 +434,9 @@ array_dealloc(PyArrayObject *self)
 {
     PyArrayObject_fields *fa = (PyArrayObject_fields *)self;
 
-    _dealloc_cached_buffer_info((PyObject*)self);
+    if (_buffer_info_free(fa->_buffer_info, (PyObject *)self) < 0) {
+        PyErr_WriteUnraisable(NULL);
+    }
 
     if (fa->weakreflist != NULL) {
         PyObject_ClearWeakRefs((PyObject *)self);
@@ -1745,7 +1747,7 @@ array_free(PyObject * v)
 NPY_NO_EXPORT PyTypeObject PyArray_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     .tp_name = "numpy.ndarray",
-    .tp_basicsize = NPY_SIZEOF_PYARRAYOBJECT,
+    .tp_basicsize = sizeof(PyArrayObject_fields),
     /* methods */
     .tp_dealloc = (destructor)array_dealloc,
     .tp_repr = (reprfunc)array_repr,