From 6f71f30e8664f624f9229313145a1d94562f57e5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 18 Jun 2024 19:34:06 -0700 Subject: [PATCH 01/10] gh-119180: Fix annotations lookup on classes with custom metaclasses See https://discuss.python.org/t/pep-749-implementing-pep-649/54974/28 and subsequent posts. --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 + Lib/test/test_type_annotations.py | 79 +++++++++++++++++-- ...-06-18-19-33-37.gh-issue-119180.JgGJVv.rst | 2 + Objects/typeobject.c | 31 ++++++-- 7 files changed, 105 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-18-19-33-37.gh-issue-119180.JgGJVv.rst diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index bc94930b85f098..cafeb1491b8a3e 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -593,6 +593,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__anext__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotate__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations_cache__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__args__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__asyncio_running_event_loop__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__await__)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 998be2ec490dd9..7d143027c3fe36 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -82,6 +82,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__anext__) STRUCT_FOR_ID(__annotate__) STRUCT_FOR_ID(__annotations__) + STRUCT_FOR_ID(__annotations_cache__) STRUCT_FOR_ID(__args__) STRUCT_FOR_ID(__asyncio_running_event_loop__) STRUCT_FOR_ID(__await__) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index bd79a7dff42f89..ddba6ba12b1b2b 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -591,6 +591,7 @@ extern "C" { INIT_ID(__anext__), \ INIT_ID(__annotate__), \ INIT_ID(__annotations__), \ + INIT_ID(__annotations_cache__), \ INIT_ID(__args__), \ INIT_ID(__asyncio_running_event_loop__), \ INIT_ID(__await__), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 7284aeb592d7ec..7a7162acd02687 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -84,6 +84,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__annotations__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__annotations_cache__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__args__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index a9be1f5aa84681..144fcf56301066 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,3 +1,4 @@ +import itertools import textwrap import types import unittest @@ -16,22 +17,22 @@ def test_lazy_create_annotations(self): # a freshly created type shouldn't have an annotations dict yet. foo = type("Foo", (), {}) for i in range(3): - self.assertFalse("__annotations__" in foo.__dict__) + self.assertFalse("__annotations_cache__" in foo.__dict__) d = foo.__annotations__ - self.assertTrue("__annotations__" in foo.__dict__) + self.assertTrue("__annotations_cache__" in foo.__dict__) self.assertEqual(foo.__annotations__, d) - self.assertEqual(foo.__dict__['__annotations__'], d) + self.assertEqual(foo.__dict__['__annotations_cache__'], d) del foo.__annotations__ def test_setting_annotations(self): foo = type("Foo", (), {}) for i in range(3): - self.assertFalse("__annotations__" in foo.__dict__) + self.assertFalse("__annotations_cache__" in foo.__dict__) d = {'a': int} foo.__annotations__ = d - self.assertTrue("__annotations__" in foo.__dict__) + self.assertTrue("__annotations_cache__" in foo.__dict__) self.assertEqual(foo.__annotations__, d) - self.assertEqual(foo.__dict__['__annotations__'], d) + self.assertEqual(foo.__dict__['__annotations_cache__'], d) del foo.__annotations__ def test_annotations_getset_raises(self): @@ -55,9 +56,9 @@ class C: a:int=3 b:str=4 self.assertEqual(C.__annotations__, {"a": int, "b": str}) - self.assertTrue("__annotations__" in C.__dict__) + self.assertTrue("__annotations_cache__" in C.__dict__) del C.__annotations__ - self.assertFalse("__annotations__" in C.__dict__) + self.assertFalse("__annotations_cache__" in C.__dict__) def test_descriptor_still_works(self): class C: @@ -270,6 +271,68 @@ def check_annotations(self, f): self.assertIs(f.__annotate__, None) +class MetaclassTests(unittest.TestCase): + def test_annotated_meta(self): + class Meta(type): + a: int + + class X(metaclass=Meta): + pass + + class Y(metaclass=Meta): + b: float + + self.assertEqual(Meta.__annotations__, {"a": int}) + self.assertEqual(Meta.__annotate__(1), {"a": int}) + + self.assertEqual(X.__annotations__, {}) + self.assertIs(X.__annotate__, None) + + self.assertEqual(Y.__annotations__, {"b": float}) + self.assertEqual(Y.__annotate__(1), {"b": float}) + + def test_ordering(self): + # Based on a sample by David Ellis + # https://discuss.python.org/t/pep-749-implementing-pep-649/54974/38 + + def make_classes(): + class Meta(type): + a: int + expected_annotations = {"a": int} + + class A(type, metaclass=Meta): + b: float + expected_annotations = {"b": float} + + class B(metaclass=A): + c: str + expected_annotations = {"c": str} + + class C(B): + expected_annotations = {} + + class D(metaclass=Meta): + expected_annotations = {} + + return Meta, A, B, C, D + + classes = make_classes() + class_count = len(classes) + for order in itertools.permutations(range(class_count), class_count): + names = ", ".join(classes[i].__name__ for i in order) + with self.subTest(names=names): + classes = make_classes() # Regenerate classes + for i in order: + classes[i].__annotations__ + for c in classes: + with self.subTest(c=c): + self.assertEqual(c.__annotations__, c.expected_annotations) + if c.expected_annotations: + self.assertEqual(c.__annotate__(1), c.expected_annotations) + else: + self.assertIs(c.__annotate__, None) + + class DeferredEvaluationTests(unittest.TestCase): def test_function(self): def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-18-19-33-37.gh-issue-119180.JgGJVv.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-18-19-33-37.gh-issue-119180.JgGJVv.rst new file mode 100644 index 00000000000000..e52aeb108dc629 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-18-19-33-37.gh-issue-119180.JgGJVv.rst @@ -0,0 +1,2 @@ +Make lookup of ``__annotate__`` and ``__annotations__`` on classes more +robust in the presence of metaclasses. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 0dcf1d399d91be..427bc5e3fb3616 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1902,7 +1902,7 @@ type_set_annotate(PyTypeObject *type, PyObject *value, void *Py_UNUSED(ignored)) return -1; } if (!Py_IsNone(value)) { - if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) { + if (PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL) == -1) { Py_DECREF(dict); PyType_Modified(type); return -1; @@ -1923,7 +1923,7 @@ type_get_annotations(PyTypeObject *type, void *context) PyObject *annotations; PyObject *dict = PyType_GetDict(type); - if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) < 0) { + if (PyDict_GetItemRef(dict, &_Py_ID(__annotations_cache__), &annotations) < 0) { Py_DECREF(dict); return NULL; } @@ -1962,7 +1962,7 @@ type_get_annotations(PyTypeObject *type, void *context) Py_DECREF(annotate); if (annotations) { int result = PyDict_SetItem( - dict, &_Py_ID(__annotations__), annotations); + dict, &_Py_ID(__annotations_cache__), annotations); if (result) { Py_CLEAR(annotations); } else { @@ -1988,10 +1988,10 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) PyObject *dict = PyType_GetDict(type); if (value != NULL) { /* set */ - result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); + result = PyDict_SetItem(dict, &_Py_ID(__annotations_cache__), value); } else { /* delete */ - result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL); + result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL); if (result == 0) { PyErr_SetString(PyExc_AttributeError, "__annotations__"); Py_DECREF(dict); @@ -4223,6 +4223,24 @@ type_new_set_classcell(PyTypeObject *type) return 0; } +static int +type_new_set_annotate(PyTypeObject *type) +{ + PyObject *dict = lookup_tp_dict(type); + // If __annotate__ is not set (i.e., the class has no annotations), + // set it to None + int result = PyDict_Contains(dict, &_Py_ID(__annotate__)); + if (result < 0) { + return -1; + } + else if (result == 0) { + if (PyDict_SetItem(dict, &_Py_ID(__annotate__), Py_None) < 0) { + return -1; + } + } + return 0; +} + static int type_new_set_classdictcell(PyTypeObject *type) { @@ -4296,6 +4314,9 @@ type_new_set_attrs(const type_new_ctx *ctx, PyTypeObject *type) if (type_new_set_classdictcell(type) < 0) { return -1; } + if (type_new_set_annotate(type) < 0) { + return -1; + } return 0; } From e468ce7fddf7bd4507ce8f229f8fe3fe7dbccc62 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 19 Jun 2024 07:20:19 -0700 Subject: [PATCH 02/10] basic annotations descriptor --- Include/internal/pycore_typeobject.h | 1 + Objects/object.c | 1 + Objects/typeobject.c | 84 ++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 32bd19d968b917..bef6a4e2bed9ce 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -226,6 +226,7 @@ extern PyObject* _Py_slot_tp_getattro(PyObject *self, PyObject *name); extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); extern PyTypeObject _PyBufferWrapper_Type; +extern PyTypeObject _PyAnnotationsDescriptor_Type; PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); diff --git a/Objects/object.c b/Objects/object.c index 16f940f46f137e..ceeaeb9bde6507 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2269,6 +2269,7 @@ static PyTypeObject* static_types[] = { &PyZip_Type, &Py_GenericAliasType, &_PyAnextAwaitable_Type, + &_PyAnnotationsDescriptor_Type, &_PyAsyncGenASend_Type, &_PyAsyncGenAThrow_Type, &_PyAsyncGenWrappedValue_Type, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 427bc5e3fb3616..82b6269a42b3d7 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -115,6 +115,9 @@ lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound); static int slot_tp_setattro(PyObject *self, PyObject *name, PyObject *value); +static PyObject * +annodescr_new(PyObject *owner); + static inline PyTypeObject * type_from_ref(PyObject *ref) @@ -4238,6 +4241,19 @@ type_new_set_annotate(PyTypeObject *type) return -1; } } + result = PyDict_Contains(dict, &_Py_ID(__annotations__)); + if (result < 0) { + return -1; + } + else if (result == 0) { + PyObject *descr = annodescr_new((PyObject *)type); + if (descr == NULL) { + return -1; + } + if (PyDict_SetItem(dict, &_Py_ID(__annotations__), descr) < 0) { + return -1; + } + } return 0; } @@ -11698,3 +11714,71 @@ PyTypeObject PySuper_Type = { PyObject_GC_Del, /* tp_free */ .tp_vectorcall = (vectorcallfunc)super_vectorcall, }; + +/* Annotations descriptor for heap types */ + +typedef struct { + PyObject_HEAD + PyObject *owner; + PyObject *annotations; +} annodescrobject; + +static PyObject * +annodescr_new(PyObject *owner) +{ + annodescrobject *ad = PyObject_GC_New(annodescrobject, &_PyAnnotationsDescriptor_Type); + if (ad == NULL) { + return NULL; + } + ad->owner = Py_NewRef(owner); + ad->annotations = NULL; + _PyObject_GC_TRACK(ad); + return (PyObject *)ad; +} + +static void +annodescr_dealloc(PyObject *self) +{ + annodescrobject *ad = (annodescrobject *)self; + + _PyObject_GC_UNTRACK(self); + Py_XDECREF(ad->owner); + Py_XDECREF(ad->annotations); + Py_TYPE(self)->tp_free(self); +} + +static int +annodescr_traverse(PyObject *self, visitproc visit, void *arg) +{ + annodescrobject *ad = (annodescrobject *)self; + + Py_VISIT(ad->owner); + Py_VISIT(ad->annotations); + + return 0; +} + +static PyObject * +annodescr_repr(PyObject *self) +{ + annodescrobject *ad = (annodescrobject *)self; + return PyUnicode_FromFormat("", ad->owner); +} + +PyDoc_STRVAR(annodescr_doc, +"Wrapper for a class's annotations dictionary."); + +PyTypeObject _PyAnnotationsDescriptor_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "AnnotationsDescriptor", + .tp_basicsize = sizeof(annodescrobject), + .tp_itemsize = 0, + .tp_dealloc = annodescr_dealloc, + .tp_traverse = annodescr_traverse, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_alloc = PyType_GenericAlloc, + .tp_new = PyType_GenericNew, + .tp_free = PyObject_GC_Del, + .tp_doc = annodescr_doc, + .tp_repr = annodescr_repr, +}; From dfcb0719d7cb1d393b3ff645a9299a8f1dfcc587 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 19 Jun 2024 08:18:44 -0700 Subject: [PATCH 03/10] Expand prototype --- Objects/typeobject.c | 229 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 197 insertions(+), 32 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 82b6269a42b3d7..882e4b7e7393bd 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1916,6 +1916,34 @@ type_set_annotate(PyTypeObject *type, PyObject *value, void *Py_UNUSED(ignored)) return 0; } +static PyObject * +type_materialize_annotations(PyTypeObject *type) +{ + PyObject *annotate = type_get_annotate(type, NULL); + if (annotate == NULL) { + return NULL; + } + if (PyCallable_Check(annotate)) { + PyObject *one = _PyLong_GetOne(); + PyObject *annotations = _PyObject_CallOneArg(annotate, one); + Py_DECREF(annotate); + if (annotations == NULL) { + return NULL; + } + if (!PyDict_Check(annotations)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(annotations)->tp_name); + Py_DECREF(annotations); + return NULL; + } + return annotations; + } + else { + Py_DECREF(annotate); + return PyDict_New(); + } +} + static PyObject * type_get_annotations(PyTypeObject *type, void *context) { @@ -1937,40 +1965,17 @@ type_get_annotations(PyTypeObject *type, void *context) } } else { - PyObject *annotate = type_get_annotate(type, NULL); - if (annotate == NULL) { + annotations = type_materialize_annotations(type); + if (annotations == NULL) { Py_DECREF(dict); return NULL; } - if (PyCallable_Check(annotate)) { - PyObject *one = _PyLong_GetOne(); - annotations = _PyObject_CallOneArg(annotate, one); - if (annotations == NULL) { - Py_DECREF(dict); - Py_DECREF(annotate); - return NULL; - } - if (!PyDict_Check(annotations)) { - PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", - Py_TYPE(annotations)->tp_name); - Py_DECREF(annotations); - Py_DECREF(annotate); - Py_DECREF(dict); - return NULL; - } - } - else { - annotations = PyDict_New(); - } - Py_DECREF(annotate); - if (annotations) { - int result = PyDict_SetItem( - dict, &_Py_ID(__annotations_cache__), annotations); - if (result) { - Py_CLEAR(annotations); - } else { - PyType_Modified(type); - } + int result = PyDict_SetItem( + dict, &_Py_ID(__annotations_cache__), annotations); + if (result) { + Py_CLEAR(annotations); + } else { + PyType_Modified(type); } } Py_DECREF(dict); @@ -11736,6 +11741,20 @@ annodescr_new(PyObject *owner) return (PyObject *)ad; } +static int +annodescr_materialize(annodescrobject *ad) +{ + if (ad->annotations == NULL) { + assert(PyType_Check(ad->owner)); + PyObject *dict = type_materialize_annotations((PyTypeObject *)ad->owner); + if (dict == NULL) { + return -1; + } + ad->annotations = dict; + } + return 0; +} + static void annodescr_dealloc(PyObject *self) { @@ -11768,6 +11787,144 @@ annodescr_repr(PyObject *self) PyDoc_STRVAR(annodescr_doc, "Wrapper for a class's annotations dictionary."); +static PyObject * +annodescr_iter(PyObject *self) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return NULL; + } + return PyObject_GetIter(ad->annotations); +} + +static Py_ssize_t +annodescr_length(PyObject *self) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return -1; + } + return PyObject_Size(ad->annotations); +} + +static PyObject * +annodescr_getitem(PyObject *self, PyObject *key) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return NULL; + } + return PyObject_GetItem(ad->annotations, key); +} + +static int +annodescr_setitem(PyObject *self, PyObject *key, PyObject *value) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return -1; + } + return PyObject_SetItem(ad->annotations, key, value); +} + +static PyMappingMethods annodescr_as_mapping = { + .mp_length = annodescr_length, + .mp_subscript = annodescr_getitem, + .mp_ass_subscript = annodescr_setitem, +}; + +static int +annodescr_contains(PyObject *self, PyObject *key) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return -1; + } + return PySequence_Contains(ad->annotations, key); +} + +static PySequenceMethods annodescr_as_sequence = { + .sq_contains = annodescr_contains, +}; + +static PyObject * +annodescr_or(PyObject *self, PyObject *other) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return NULL; + } + return PyNumber_Or(ad->annotations, other); +} + +static PyObject * +annodescr_inplace_or(PyObject *self, PyObject *other) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return NULL; + } + return PyNumber_InPlaceOr(ad->annotations, other); +} + +static PyNumberMethods annodescr_as_number = { + .nb_or = annodescr_or, + .nb_inplace_or = annodescr_inplace_or, +}; + +static PyObject * +annodescr_richcompare(PyObject *self, PyObject *other, int op) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return NULL; + } + return PyObject_RichCompare(ad->annotations, other, op); +} + +#define MAKE_DICT_WRAPPER(name) \ + PyObject *annodescr_ ## name(PyObject *self, \ + PyObject *args, \ + PyObject *kwargs) \ + { \ + annodescrobject *ad = (annodescrobject *)self; \ + if (annodescr_materialize(ad) < 0) { \ + return NULL; \ + } \ + PyObject *method = PyObject_GetAttrString(ad->annotations, #name); \ + if (method == NULL) { \ + return NULL; \ + } \ + return PyObject_Call(method, args, kwargs); \ + } + +MAKE_DICT_WRAPPER(get) +MAKE_DICT_WRAPPER(setdefault) +MAKE_DICT_WRAPPER(pop) +MAKE_DICT_WRAPPER(popitem) +MAKE_DICT_WRAPPER(keys) +MAKE_DICT_WRAPPER(values) +MAKE_DICT_WRAPPER(items) +MAKE_DICT_WRAPPER(update) +MAKE_DICT_WRAPPER(clear) +MAKE_DICT_WRAPPER(copy) + +#undef MAKE_DICT_WRAPPER + +static PyMethodDef annodescr_methods[] = { + {"get", (PyCFunction)annodescr_get, METH_VARARGS | METH_KEYWORDS, NULL}, + {"setdefault", (PyCFunction)annodescr_setdefault, METH_VARARGS | METH_KEYWORDS, NULL}, + {"pop", (PyCFunction)annodescr_pop, METH_VARARGS | METH_KEYWORDS, NULL}, + {"popitem", (PyCFunction)annodescr_popitem, METH_VARARGS | METH_KEYWORDS, NULL}, + {"keys", (PyCFunction)annodescr_keys, METH_VARARGS | METH_KEYWORDS, NULL}, + {"values", (PyCFunction)annodescr_values, METH_VARARGS | METH_KEYWORDS, NULL}, + {"items", (PyCFunction)annodescr_items, METH_VARARGS | METH_KEYWORDS, NULL}, + {"update", (PyCFunction)annodescr_update, METH_VARARGS | METH_KEYWORDS, NULL}, + {"clear", (PyCFunction)annodescr_clear, METH_VARARGS | METH_KEYWORDS, NULL}, + {"copy", (PyCFunction)annodescr_copy, METH_VARARGS | METH_KEYWORDS, NULL}, + {NULL, NULL}, +}; + PyTypeObject _PyAnnotationsDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "AnnotationsDescriptor", @@ -11775,10 +11932,18 @@ PyTypeObject _PyAnnotationsDescriptor_Type = { .tp_itemsize = 0, .tp_dealloc = annodescr_dealloc, .tp_traverse = annodescr_traverse, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MAPPING, .tp_alloc = PyType_GenericAlloc, .tp_new = PyType_GenericNew, .tp_free = PyObject_GC_Del, .tp_doc = annodescr_doc, .tp_repr = annodescr_repr, + .tp_iter = annodescr_iter, + .tp_as_mapping = &annodescr_as_mapping, + .tp_as_sequence = &annodescr_as_sequence, + .tp_as_number = &annodescr_as_number, + .tp_hash = PyObject_HashNotImplemented, + .tp_getattro = PyObject_GenericGetAttr, + .tp_richcompare = annodescr_richcompare, + .tp_methods = annodescr_methods, }; From c0a2668f2170ec8b6057b9d58cf8830b52ddc2d7 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 20 Jun 2024 18:15:32 -0700 Subject: [PATCH 04/10] add get/set --- .../pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - .../internal/pycore_runtime_init_generated.h | 1 - .../internal/pycore_unicodeobject_generated.h | 3 -- Objects/typeobject.c | 34 ++++++++++++++++--- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index cafeb1491b8a3e..bc94930b85f098 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -593,7 +593,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__anext__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotate__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations__)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations_cache__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__args__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__asyncio_running_event_loop__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__await__)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 7d143027c3fe36..998be2ec490dd9 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -82,7 +82,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(__anext__) STRUCT_FOR_ID(__annotate__) STRUCT_FOR_ID(__annotations__) - STRUCT_FOR_ID(__annotations_cache__) STRUCT_FOR_ID(__args__) STRUCT_FOR_ID(__asyncio_running_event_loop__) STRUCT_FOR_ID(__await__) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index ddba6ba12b1b2b..bd79a7dff42f89 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -591,7 +591,6 @@ extern "C" { INIT_ID(__anext__), \ INIT_ID(__annotate__), \ INIT_ID(__annotations__), \ - INIT_ID(__annotations_cache__), \ INIT_ID(__args__), \ INIT_ID(__asyncio_running_event_loop__), \ INIT_ID(__await__), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 7a7162acd02687..7284aeb592d7ec 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -84,9 +84,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__annotations__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(__annotations_cache__); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__args__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 221079bff4b4c3..e757e5434c8817 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1880,7 +1880,7 @@ type_set_annotate(PyTypeObject *type, PyObject *value, void *Py_UNUSED(ignored)) return -1; } if (!Py_IsNone(value)) { - if (PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL) == -1) { + if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) { Py_DECREF(dict); PyType_Modified(type); return -1; @@ -1929,7 +1929,7 @@ type_get_annotations(PyTypeObject *type, void *context) PyObject *annotations; PyObject *dict = PyType_GetDict(type); - if (PyDict_GetItemRef(dict, &_Py_ID(__annotations_cache__), &annotations) < 0) { + if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) < 0) { Py_DECREF(dict); return NULL; } @@ -1946,7 +1946,7 @@ type_get_annotations(PyTypeObject *type, void *context) return NULL; } int result = PyDict_SetItem( - dict, &_Py_ID(__annotations_cache__), annotations); + dict, &_Py_ID(__annotations__), annotations); if (result) { Py_CLEAR(annotations); } else { @@ -1971,10 +1971,10 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) PyObject *dict = PyType_GetDict(type); if (value != NULL) { /* set */ - result = PyDict_SetItem(dict, &_Py_ID(__annotations_cache__), value); + result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); } else { /* delete */ - result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL); + result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL); if (result == 0) { PyErr_SetString(PyExc_AttributeError, "__annotations__"); Py_DECREF(dict); @@ -11889,6 +11889,28 @@ static PyMethodDef annodescr_methods[] = { {NULL, NULL}, }; +static PyObject * +annodescr_descr_get(PyObject *self, PyObject *obj, PyObject *type) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return NULL; + } + return Py_NewRef(ad->annotations); +} + +static int +annodescr_descr_set(PyObject *self, PyObject *obj, PyObject *value) +{ + annodescrobject *ad = (annodescrobject *)self; + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete __annotations__"); + return -1; + } + Py_XSETREF(ad->annotations, value); + return 0; +} + PyTypeObject _PyAnnotationsDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "AnnotationsDescriptor", @@ -11910,4 +11932,6 @@ PyTypeObject _PyAnnotationsDescriptor_Type = { .tp_getattro = PyObject_GenericGetAttr, .tp_richcompare = annodescr_richcompare, .tp_methods = annodescr_methods, + .tp_descr_get = annodescr_descr_get, + .tp_descr_set = annodescr_descr_set, }; From 20fb1445b975f44a78f51ab43aaa5e7aa644d79d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 20 Jun 2024 18:18:44 -0700 Subject: [PATCH 05/10] no more __annotations_cache__ --- Lib/test/test_type_annotations.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 144fcf56301066..2b40b6b0a82573 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -17,22 +17,22 @@ def test_lazy_create_annotations(self): # a freshly created type shouldn't have an annotations dict yet. foo = type("Foo", (), {}) for i in range(3): - self.assertFalse("__annotations_cache__" in foo.__dict__) + self.assertFalse("__annotations__" in foo.__dict__) d = foo.__annotations__ - self.assertTrue("__annotations_cache__" in foo.__dict__) + self.assertTrue("__annotations__" in foo.__dict__) self.assertEqual(foo.__annotations__, d) - self.assertEqual(foo.__dict__['__annotations_cache__'], d) + self.assertEqual(foo.__dict__['__annotations__'], d) del foo.__annotations__ def test_setting_annotations(self): foo = type("Foo", (), {}) for i in range(3): - self.assertFalse("__annotations_cache__" in foo.__dict__) + self.assertFalse("__annotations__" in foo.__dict__) d = {'a': int} foo.__annotations__ = d - self.assertTrue("__annotations_cache__" in foo.__dict__) + self.assertTrue("__annotations__" in foo.__dict__) self.assertEqual(foo.__annotations__, d) - self.assertEqual(foo.__dict__['__annotations_cache__'], d) + self.assertEqual(foo.__dict__['__annotations__'], d) del foo.__annotations__ def test_annotations_getset_raises(self): @@ -56,9 +56,9 @@ class C: a:int=3 b:str=4 self.assertEqual(C.__annotations__, {"a": int, "b": str}) - self.assertTrue("__annotations_cache__" in C.__dict__) + self.assertTrue("__annotations__" in C.__dict__) del C.__annotations__ - self.assertFalse("__annotations_cache__" in C.__dict__) + self.assertFalse("__annotations__" in C.__dict__) def test_descriptor_still_works(self): class C: From 832f6f324704205552c2aefb3c3cbd9ff6973869 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 20 Jun 2024 18:52:04 -0700 Subject: [PATCH 06/10] Don't add __set__ --- Lib/test/test_type_annotations.py | 5 ----- Objects/typeobject.c | 13 ------------- 2 files changed, 18 deletions(-) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 2b40b6b0a82573..04c78231229bc9 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -12,12 +12,8 @@ class TypeAnnotationTests(unittest.TestCase): def test_lazy_create_annotations(self): - # type objects lazy create their __annotations__ dict on demand. - # the annotations dict is stored in type.__dict__. - # a freshly created type shouldn't have an annotations dict yet. foo = type("Foo", (), {}) for i in range(3): - self.assertFalse("__annotations__" in foo.__dict__) d = foo.__annotations__ self.assertTrue("__annotations__" in foo.__dict__) self.assertEqual(foo.__annotations__, d) @@ -27,7 +23,6 @@ def test_lazy_create_annotations(self): def test_setting_annotations(self): foo = type("Foo", (), {}) for i in range(3): - self.assertFalse("__annotations__" in foo.__dict__) d = {'a': int} foo.__annotations__ = d self.assertTrue("__annotations__" in foo.__dict__) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index e757e5434c8817..4fcbe8db2fd61a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -11899,18 +11899,6 @@ annodescr_descr_get(PyObject *self, PyObject *obj, PyObject *type) return Py_NewRef(ad->annotations); } -static int -annodescr_descr_set(PyObject *self, PyObject *obj, PyObject *value) -{ - annodescrobject *ad = (annodescrobject *)self; - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "cannot delete __annotations__"); - return -1; - } - Py_XSETREF(ad->annotations, value); - return 0; -} - PyTypeObject _PyAnnotationsDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "AnnotationsDescriptor", @@ -11933,5 +11921,4 @@ PyTypeObject _PyAnnotationsDescriptor_Type = { .tp_richcompare = annodescr_richcompare, .tp_methods = annodescr_methods, .tp_descr_get = annodescr_descr_get, - .tp_descr_set = annodescr_descr_set, }; From 1d3abff2f03a9b59051ad96f44988a76cd2150c1 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 20 Jun 2024 19:04:54 -0700 Subject: [PATCH 07/10] fix leak --- Objects/typeobject.c | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4fcbe8db2fd61a..539318bf726a5f 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4231,8 +4231,10 @@ type_new_set_annotate(PyTypeObject *type) return -1; } if (PyDict_SetItem(dict, &_Py_ID(__annotations__), descr) < 0) { + Py_DECREF(descr); return -1; } + Py_DECREF(descr); } return 0; } @@ -11723,11 +11725,13 @@ static void annodescr_dealloc(PyObject *self) { annodescrobject *ad = (annodescrobject *)self; + PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); Py_XDECREF(ad->owner); Py_XDECREF(ad->annotations); - Py_TYPE(self)->tp_free(self); + tp->tp_free(self); + Py_DECREF(tp); } static int @@ -11741,6 +11745,16 @@ annodescr_traverse(PyObject *self, visitproc visit, void *arg) return 0; } +static int +annodescr_clear(PyObject *self) +{ + annodescrobject *ad = (annodescrobject *)self; + + Py_CLEAR(ad->annotations); + Py_CLEAR(ad->owner); + return 0; +} + static PyObject * annodescr_repr(PyObject *self) { @@ -11847,7 +11861,7 @@ annodescr_richcompare(PyObject *self, PyObject *other, int op) } #define MAKE_DICT_WRAPPER(name) \ - PyObject *annodescr_ ## name(PyObject *self, \ + PyObject *annodescr_wrap_ ## name(PyObject *self, \ PyObject *args, \ PyObject *kwargs) \ { \ @@ -11876,16 +11890,16 @@ MAKE_DICT_WRAPPER(copy) #undef MAKE_DICT_WRAPPER static PyMethodDef annodescr_methods[] = { - {"get", (PyCFunction)annodescr_get, METH_VARARGS | METH_KEYWORDS, NULL}, - {"setdefault", (PyCFunction)annodescr_setdefault, METH_VARARGS | METH_KEYWORDS, NULL}, - {"pop", (PyCFunction)annodescr_pop, METH_VARARGS | METH_KEYWORDS, NULL}, - {"popitem", (PyCFunction)annodescr_popitem, METH_VARARGS | METH_KEYWORDS, NULL}, - {"keys", (PyCFunction)annodescr_keys, METH_VARARGS | METH_KEYWORDS, NULL}, - {"values", (PyCFunction)annodescr_values, METH_VARARGS | METH_KEYWORDS, NULL}, - {"items", (PyCFunction)annodescr_items, METH_VARARGS | METH_KEYWORDS, NULL}, - {"update", (PyCFunction)annodescr_update, METH_VARARGS | METH_KEYWORDS, NULL}, - {"clear", (PyCFunction)annodescr_clear, METH_VARARGS | METH_KEYWORDS, NULL}, - {"copy", (PyCFunction)annodescr_copy, METH_VARARGS | METH_KEYWORDS, NULL}, + {"get", (PyCFunction)annodescr_wrap_get, METH_VARARGS | METH_KEYWORDS, NULL}, + {"setdefault", (PyCFunction)annodescr_wrap_setdefault, METH_VARARGS | METH_KEYWORDS, NULL}, + {"pop", (PyCFunction)annodescr_wrap_pop, METH_VARARGS | METH_KEYWORDS, NULL}, + {"popitem", (PyCFunction)annodescr_wrap_popitem, METH_VARARGS | METH_KEYWORDS, NULL}, + {"keys", (PyCFunction)annodescr_wrap_keys, METH_VARARGS | METH_KEYWORDS, NULL}, + {"values", (PyCFunction)annodescr_wrap_values, METH_VARARGS | METH_KEYWORDS, NULL}, + {"items", (PyCFunction)annodescr_wrap_items, METH_VARARGS | METH_KEYWORDS, NULL}, + {"update", (PyCFunction)annodescr_wrap_update, METH_VARARGS | METH_KEYWORDS, NULL}, + {"clear", (PyCFunction)annodescr_wrap_clear, METH_VARARGS | METH_KEYWORDS, NULL}, + {"copy", (PyCFunction)annodescr_wrap_copy, METH_VARARGS | METH_KEYWORDS, NULL}, {NULL, NULL}, }; @@ -11906,6 +11920,7 @@ PyTypeObject _PyAnnotationsDescriptor_Type = { .tp_itemsize = 0, .tp_dealloc = annodescr_dealloc, .tp_traverse = annodescr_traverse, + .tp_clear = annodescr_clear, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MAPPING, .tp_alloc = PyType_GenericAlloc, .tp_new = PyType_GenericNew, From 9969e99ce96bd4ed42e9362688cbff5c7c3b370d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 20 Jun 2024 19:18:09 -0700 Subject: [PATCH 08/10] fix test_descr --- Lib/test/test_descr.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index b0f86317bfecf6..2dc559a5656ac4 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5123,7 +5123,8 @@ def test_iter_keys(self): self.assertNotIsInstance(it, list) keys = list(it) keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + self.assertEqual(keys, ['__annotate__', '__annotations__', + '__dict__', '__doc__', '__firstlineno__', '__module__', '__static_attributes__', '__weakref__', 'meth']) @@ -5135,7 +5136,7 @@ def test_iter_values(self): it = self.C.__dict__.values() self.assertNotIsInstance(it, list) values = list(it) - self.assertEqual(len(values), 7) + self.assertEqual(len(values), 9) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') @@ -5145,7 +5146,8 @@ def test_iter_items(self): self.assertNotIsInstance(it, list) keys = [item[0] for item in it] keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + self.assertEqual(keys, ['__annotate__', '__annotations__', + '__dict__', '__doc__', '__firstlineno__', '__module__', '__static_attributes__', '__weakref__', 'meth']) From b845c0e1fc150232c826090e7665636116e94efb Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 20 Jun 2024 19:30:32 -0700 Subject: [PATCH 09/10] fix tests --- Lib/doctest.py | 1 + Lib/pydoc.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/doctest.py b/Lib/doctest.py index ea7d275c91db04..3c209c3e6c5895 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1067,6 +1067,7 @@ def _find(self, tests, obj, name, module, source_lines, globs, seen): # Recurse to methods, properties, and nested classes. if ((inspect.isroutine(val) or inspect.isclass(val) or isinstance(val, property)) and + valname != '__annotations__' and self._from_module(module, val)): valname = '%s.%s' % (name, valname) self._find(tests, val, valname, module, source_lines, diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 768c3dcb11ec59..0f5fe090f33331 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -328,8 +328,10 @@ def visiblename(name, all=None, obj=None): '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', '__path__', '__qualname__', '__slots__', '__version__', - '__static_attributes__', '__firstlineno__'}: + '__static_attributes__', '__firstlineno__', '__annotations__'}: return 0 + if name == '__annotate__' and getattr(obj, name, None) is None: + return False # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 # Namedtuples have public fields and methods with a single leading underscore From 73eb018d0463cf24cf81a0a6d322418a7660849a Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 20 Jun 2024 20:28:01 -0700 Subject: [PATCH 10/10] fix smelly --- Objects/typeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 539318bf726a5f..d1233d7ec28baa 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -11861,7 +11861,7 @@ annodescr_richcompare(PyObject *self, PyObject *other, int op) } #define MAKE_DICT_WRAPPER(name) \ - PyObject *annodescr_wrap_ ## name(PyObject *self, \ + static PyObject *annodescr_wrap_ ## name(PyObject *self, \ PyObject *args, \ PyObject *kwargs) \ { \