From 6eb35625e42a1ae4a5806142a6dba7f5ecd63c6d Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 2 Nov 2023 14:37:24 +1100 Subject: [PATCH 01/29] Initial design for shareable tuples --- Lib/test/test__xxsubinterpreters.py | 6 +++ Lib/test/test_interpreters.py | 1 + Python/crossinterp.c | 63 +++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index ae7dfa19acc519..270dc0e8d13014 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -105,6 +105,7 @@ def test_default_shareables(self): True, False, 100.0, + (), ] for obj in shareables: with self.subTest(obj): @@ -195,6 +196,11 @@ def test_bool(self): def test_float(self): self._assert_values([0.0, 1.1, -1.0, 0.12345678, -0.12345678]) + def test_tuple(self): + self._assert_values([(), (1,), ("hello", "world", ), (1, True, "hello")]) + # Test nesting + self._assert_values([((1,),), ((1, 2), (3, 4)), ((1, 2), (3, 4), (5, 6))]) + class ModuleTests(TestBase): diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 74f86088b45590..9aa111bd5712f8 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -781,6 +781,7 @@ def test_default_shareables(self): True, False, 100.0, + (), ] for obj in shareables: with self.subTest(obj): diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 5e3bf4cb6aa238..394936cb89fdf5 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -713,6 +713,64 @@ _bool_shared(PyThreadState *tstate, PyObject *obj, return 0; } +struct _shared_tuple_data { + Py_ssize_t len; + _PyCrossInterpreterData ** data; +}; + +static PyObject * +_new_tuple_object(_PyCrossInterpreterData *data) +{ + struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data->data); + PyObject *tuple = PyTuple_New(shared->len); + + for (Py_ssize_t i = 0; i < shared->len; i++) { + PyObject *item = _PyCrossInterpreterData_NewObject(shared->data[i]); + PyTuple_SET_ITEM(tuple, i, item); + } + return tuple; +} + +void _tuple_shared_free(void* data) { + struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data); + for (Py_ssize_t i = 0; i < shared->len; i++) { + _PyCrossInterpreterData_Release(shared->data[i]); + } + PyMem_Free(shared->data); + PyMem_RawFree(shared); +} + +static int +_tuple_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_tuple_data), obj, + _new_tuple_object + ) < 0) + { + return -1; + } + struct _shared_tuple_data *shared = (struct _shared_tuple_data *)data->data; + shared->len = PyTuple_GET_SIZE(obj); + shared->data = (_PyCrossInterpreterData **) PyMem_Calloc(shared->len, sizeof(_PyCrossInterpreterData *)); + if (shared->data == NULL) { + // Set no memory exception? + return -1; + } + + for (Py_ssize_t i = 0; i < shared->len; i++) { + shared->data[i] = _PyCrossInterpreterData_New(); + // TODO: Where does this field get free'ed? + PyObject *item = PyTuple_GET_ITEM(obj, i); + if (_PyObject_GetCrossInterpreterData(item, shared->data[i]) != 0) { + return -1; + } + } + data->free = _tuple_shared_free; + return 0; +} + static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) { @@ -745,6 +803,11 @@ _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) if (_xidregistry_add_type(xidregistry, &PyFloat_Type, _float_shared) != 0) { Py_FatalError("could not register float for cross-interpreter sharing"); } + + // tuple + if (_xidregistry_add_type(xidregistry, &PyTuple_Type, _tuple_shared) != 0) { + Py_FatalError("could not register tuple for cross-interpreter sharing"); + } } /* registry lifecycle */ From d3a5cfcd1fe9089e2fd5d5aa2d9b31b1a45b698e Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 2 Nov 2023 14:41:31 +1100 Subject: [PATCH 02/29] Add memory error --- Python/crossinterp.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 394936cb89fdf5..5119140dd41f36 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -755,13 +755,12 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, shared->len = PyTuple_GET_SIZE(obj); shared->data = (_PyCrossInterpreterData **) PyMem_Calloc(shared->len, sizeof(_PyCrossInterpreterData *)); if (shared->data == NULL) { - // Set no memory exception? + PyErr_Format(PyExc_MemoryError, "not enough memory"); return -1; } for (Py_ssize_t i = 0; i < shared->len; i++) { shared->data[i] = _PyCrossInterpreterData_New(); - // TODO: Where does this field get free'ed? PyObject *item = PyTuple_GET_ITEM(obj, i); if (_PyObject_GetCrossInterpreterData(item, shared->data[i]) != 0) { return -1; From 439b65207aba9455798ef302d761444b08a2f831 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 2 Nov 2023 14:58:26 +1100 Subject: [PATCH 03/29] Add a check for XID allocation failures --- Python/crossinterp.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 5119140dd41f36..6fbd75469c23ae 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -761,6 +761,10 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, for (Py_ssize_t i = 0; i < shared->len; i++) { shared->data[i] = _PyCrossInterpreterData_New(); + if (shared->data[i] == NULL){ + PyMem_Free(shared->data); + return -1; // PyErr_NoMemory already set + } PyObject *item = PyTuple_GET_ITEM(obj, i); if (_PyObject_GetCrossInterpreterData(item, shared->data[i]) != 0) { return -1; From 2d8602d6c810f9507ecf044e3ba44d9f35f8a80d Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 2 Nov 2023 15:01:06 +1100 Subject: [PATCH 04/29] Add blurb --- .../2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst new file mode 100644 index 00000000000000..f45faa7a28e121 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst @@ -0,0 +1,2 @@ +Support for sharing tuples between interpreters using the cross-interpreter +API From 3711bb54347bad1c607ca4e3e0142b68e3e52241 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 2 Nov 2023 15:04:00 +1100 Subject: [PATCH 05/29] tweaks for pep7 --- Python/crossinterp.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 6fbd75469c23ae..db62dcde2b5495 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -731,7 +731,9 @@ _new_tuple_object(_PyCrossInterpreterData *data) return tuple; } -void _tuple_shared_free(void* data) { +static void +_tuple_shared_free(void* data) +{ struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data); for (Py_ssize_t i = 0; i < shared->len; i++) { _PyCrossInterpreterData_Release(shared->data[i]); From 7072d36fc434547ccaeb7a0df412b1bacb749a50 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 3 Nov 2023 09:27:27 +1100 Subject: [PATCH 06/29] Add tests for non shareable tuples --- Lib/test/test__xxsubinterpreters.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 270dc0e8d13014..887b34588775e2 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -201,6 +201,24 @@ def test_tuple(self): # Test nesting self._assert_values([((1,),), ((1, 2), (3, 4)), ((1, 2), (3, 4), (5, 6))]) + def test_tuples_containing_non_shareable_types(self): + non_shareables = [ + Exception(), + object(), + ] + for s in non_shareables: + value = tuple([0, 1., s]) + with self.subTest(repr(value)): + # XXX Assert the NotShareableError when it is exported + with self.assertRaises(Exception): + _testinternalcapi.get_crossinterp_data(value) + # Check nested as well + value = tuple([0, 1., (s,)]) + with self.subTest("nested " + repr(value)): + # XXX Assert the NotShareableError when it is exported + with self.assertRaises(Exception): + _testinternalcapi.get_crossinterp_data(value) + class ModuleTests(TestBase): From d6f386fcdd86b287ba8cf6857912fb3a9cc70e8d Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 3 Nov 2023 10:01:54 +1100 Subject: [PATCH 07/29] fix patchcheck --- Lib/test/test__xxsubinterpreters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 887b34588775e2..cdf752ad65ec07 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -211,13 +211,13 @@ def test_tuples_containing_non_shareable_types(self): with self.subTest(repr(value)): # XXX Assert the NotShareableError when it is exported with self.assertRaises(Exception): - _testinternalcapi.get_crossinterp_data(value) + _testinternalcapi.get_crossinterp_data(value) # Check nested as well value = tuple([0, 1., (s,)]) with self.subTest("nested " + repr(value)): # XXX Assert the NotShareableError when it is exported with self.assertRaises(Exception): - _testinternalcapi.get_crossinterp_data(value) + _testinternalcapi.get_crossinterp_data(value) class ModuleTests(TestBase): From 08cdfcbe555206961c9a0f5855cf575c42767ac9 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 3 Nov 2023 10:28:00 +1100 Subject: [PATCH 08/29] Update Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst Co-authored-by: Pieter Eendebak --- .../2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst index f45faa7a28e121..39f26093fc4f7a 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst @@ -1,2 +1,2 @@ -Support for sharing tuples between interpreters using the cross-interpreter +Add support for sharing tuples between interpreters using the cross-interpreter API From 654689e71d1f43ed1b9f3e03d83ca5a6fe55e886 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 3 Nov 2023 10:28:11 +1100 Subject: [PATCH 09/29] Update Lib/test/test_interpreters.py Co-authored-by: Pieter Eendebak --- Lib/test/test_interpreters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 9aa111bd5712f8..2068eeb9cc73d1 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -782,6 +782,7 @@ def test_default_shareables(self): False, 100.0, (), + (1, True), ] for obj in shareables: with self.subTest(obj): From 994fab03a752dcde40b4fcc372c462eeb9cf092a Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 3 Nov 2023 10:35:07 +1100 Subject: [PATCH 10/29] Add two fault cases --- Python/crossinterp.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index db62dcde2b5495..ecd4cab3ba1878 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -723,9 +723,16 @@ _new_tuple_object(_PyCrossInterpreterData *data) { struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data->data); PyObject *tuple = PyTuple_New(shared->len); + if (tuple == NULL) { + return NULL; + } for (Py_ssize_t i = 0; i < shared->len; i++) { PyObject *item = _PyCrossInterpreterData_NewObject(shared->data[i]); + if (item == NULL){ + Py_DECREF(tuple); + return NULL; + } PyTuple_SET_ITEM(tuple, i, item); } return tuple; From 302dc8df38d2cd37f5abceeeb8963f0831b6e961 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 11:14:10 +0900 Subject: [PATCH 11/29] Update test__xxsubinterpreters.py Co-authored-by: Eric Snow --- Lib/test/test__xxsubinterpreters.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index cdf752ad65ec07..60501ee8e5d4bb 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -199,7 +199,11 @@ def test_float(self): def test_tuple(self): self._assert_values([(), (1,), ("hello", "world", ), (1, True, "hello")]) # Test nesting - self._assert_values([((1,),), ((1, 2), (3, 4)), ((1, 2), (3, 4), (5, 6))]) + self._assert_values([ + ((1,),), + ((1, 2), (3, 4)), + ((1, 2), (3, 4), (5, 6)), + ]) def test_tuples_containing_non_shareable_types(self): non_shareables = [ From 86c1494295b3c8a8bff85e5af56807d6f131bbb8 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 11:14:24 +0900 Subject: [PATCH 12/29] Update test__xxsubinterpreters.py Co-authored-by: Eric Snow --- Lib/test/test__xxsubinterpreters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 60501ee8e5d4bb..0437c24aff1938 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -211,7 +211,7 @@ def test_tuples_containing_non_shareable_types(self): object(), ] for s in non_shareables: - value = tuple([0, 1., s]) + value = tuple([0, 1.0, s]) with self.subTest(repr(value)): # XXX Assert the NotShareableError when it is exported with self.assertRaises(Exception): From 3b30b77963e99827e1cb3b664d48e8a52173c74b Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 11:14:43 +0900 Subject: [PATCH 13/29] Update test__xxsubinterpreters.py Co-authored-by: Eric Snow --- Lib/test/test__xxsubinterpreters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 0437c24aff1938..8f372ca46f4663 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -214,7 +214,7 @@ def test_tuples_containing_non_shareable_types(self): value = tuple([0, 1.0, s]) with self.subTest(repr(value)): # XXX Assert the NotShareableError when it is exported - with self.assertRaises(Exception): + with self.assertRaises(ValueError): _testinternalcapi.get_crossinterp_data(value) # Check nested as well value = tuple([0, 1., (s,)]) From 9abda96e885267e51b88deeafc4dda1326b236ae Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 11:14:50 +0900 Subject: [PATCH 14/29] Update test__xxsubinterpreters.py Co-authored-by: Eric Snow --- Lib/test/test__xxsubinterpreters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 8f372ca46f4663..dd1f4957542161 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -220,7 +220,7 @@ def test_tuples_containing_non_shareable_types(self): value = tuple([0, 1., (s,)]) with self.subTest("nested " + repr(value)): # XXX Assert the NotShareableError when it is exported - with self.assertRaises(Exception): + with self.assertRaises(ValueError): _testinternalcapi.get_crossinterp_data(value) From 82f4836b759d2306dbb8cca5e9e4e66fb7c85a0e Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 11:15:05 +0900 Subject: [PATCH 15/29] Update test__xxsubinterpreters.py Co-authored-by: Eric Snow --- Lib/test/test__xxsubinterpreters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index dd1f4957542161..97314ddbb55ec8 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -105,7 +105,7 @@ def test_default_shareables(self): True, False, 100.0, - (), + (1, ('spam', 'eggs')), ] for obj in shareables: with self.subTest(obj): From 0fc4eab32db35e75219d727832a90a0fba11998e Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 11:15:23 +0900 Subject: [PATCH 16/29] Update test_interpreters.py Co-authored-by: Eric Snow --- Lib/test/test_interpreters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 2068eeb9cc73d1..7c030bcf0321cd 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -782,7 +782,7 @@ def test_default_shareables(self): False, 100.0, (), - (1, True), + (1, ('spam', 'eggs'), True), ] for obj in shareables: with self.subTest(obj): From e234be8decf569302df8ed52369ed0f6a34b0c51 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 11:15:36 +0900 Subject: [PATCH 17/29] Update 2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst Co-authored-by: Eric Snow --- .../2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst index 39f26093fc4f7a..3a75d5e36841b4 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst @@ -1,2 +1,2 @@ Add support for sharing tuples between interpreters using the cross-interpreter -API +API. Patch by Anthony Shaw. From 1b3bb14006589cb808abd724e81ef6ab6ca349d2 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 11:15:48 +0900 Subject: [PATCH 18/29] Update crossinterp.c Co-authored-by: Eric Snow --- Python/crossinterp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index ecd4cab3ba1878..492464eb71f4eb 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -715,7 +715,7 @@ _bool_shared(PyThreadState *tstate, PyObject *obj, struct _shared_tuple_data { Py_ssize_t len; - _PyCrossInterpreterData ** data; + _PyCrossInterpreterData **data; }; static PyObject * From e70dba953392e094875d07d6bbb3179ece85c417 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 11:16:06 +0900 Subject: [PATCH 19/29] Update crossinterp.c Co-authored-by: Eric Snow --- Python/crossinterp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 492464eb71f4eb..01393c85f1a38b 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -764,7 +764,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, shared->len = PyTuple_GET_SIZE(obj); shared->data = (_PyCrossInterpreterData **) PyMem_Calloc(shared->len, sizeof(_PyCrossInterpreterData *)); if (shared->data == NULL) { - PyErr_Format(PyExc_MemoryError, "not enough memory"); + PyErr_NoMemory(); return -1; } From c8830407b9be4099c7ab38b150942c48bc210eab Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 11:16:48 +0900 Subject: [PATCH 20/29] Update crossinterp.c Co-authored-by: Eric Snow --- Python/crossinterp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 01393c85f1a38b..7ae0fcf1c68620 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -771,7 +771,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, for (Py_ssize_t i = 0; i < shared->len; i++) { shared->data[i] = _PyCrossInterpreterData_New(); if (shared->data[i] == NULL){ - PyMem_Free(shared->data); + _tuple_shared_free(shared); return -1; // PyErr_NoMemory already set } PyObject *item = PyTuple_GET_ITEM(obj, i); From 7acf36f0ab59b73206ee97ea01616117600c59c1 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 11:16:56 +0900 Subject: [PATCH 21/29] Update crossinterp.c Co-authored-by: Eric Snow --- Python/crossinterp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 7ae0fcf1c68620..e1b69e722b7e06 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -776,6 +776,8 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, } PyObject *item = PyTuple_GET_ITEM(obj, i); if (_PyObject_GetCrossInterpreterData(item, shared->data[i]) != 0) { + PyMem_RawFree(shared->data[i]); + _tuple_shared_free(shared); return -1; } } From beae22acb3081110a57476134d0ca4f767d17d57 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 11:17:33 +0900 Subject: [PATCH 22/29] Update crossinterp.c Co-authored-by: Eric Snow --- Python/crossinterp.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index e1b69e722b7e06..ca203c2bcae190 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -753,6 +753,10 @@ static int _tuple_shared(PyThreadState *tstate, PyObject *obj, _PyCrossInterpreterData *data) { + Py_ssize_t len = PyTuple_GET_SIZE(obj); + if (len < 0) { + return -1; + } if (_PyCrossInterpreterData_InitWithSize( data, tstate->interp, sizeof(struct _shared_tuple_data), obj, _new_tuple_object @@ -761,7 +765,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, return -1; } struct _shared_tuple_data *shared = (struct _shared_tuple_data *)data->data; - shared->len = PyTuple_GET_SIZE(obj); + shared->len = len; shared->data = (_PyCrossInterpreterData **) PyMem_Calloc(shared->len, sizeof(_PyCrossInterpreterData *)); if (shared->data == NULL) { PyErr_NoMemory(); From 063c8f7eb7f3d22b50d69213e5e4444ac5c596b7 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 11:18:15 +0900 Subject: [PATCH 23/29] Update crossinterp.c Co-authored-by: Eric Snow --- Python/crossinterp.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index ca203c2bcae190..1c0da644ed9206 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -743,7 +743,9 @@ _tuple_shared_free(void* data) { struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data); for (Py_ssize_t i = 0; i < shared->len; i++) { - _PyCrossInterpreterData_Release(shared->data[i]); + if (shared->data[i] != NULL) { + _PyCrossInterpreterData_Release(shared->data[i]); + } } PyMem_Free(shared->data); PyMem_RawFree(shared); From 0bd3b88b66561171cd79478fc007d7ce9b8a5b6c Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sat, 4 Nov 2023 19:01:45 +1100 Subject: [PATCH 24/29] Update the previous two blurbs to match this one --- .../2023-10-29-11-35-21.gh-issue-111435.ageUWQ.rst | 3 ++- .../2023-10-29-12-33-33.gh-issue-111438.bHTLLl.rst | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-29-11-35-21.gh-issue-111435.ageUWQ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-29-11-35-21.gh-issue-111435.ageUWQ.rst index 95044dc3b96660..034615581b6789 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-10-29-11-35-21.gh-issue-111435.ageUWQ.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-29-11-35-21.gh-issue-111435.ageUWQ.rst @@ -1 +1,2 @@ -Added support for sharing of bool type with interpreters API. +Add support for sharing of True and False between interpreters using the cross-interpreter +API. Patch by Anthony Shaw. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-29-12-33-33.gh-issue-111438.bHTLLl.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-29-12-33-33.gh-issue-111438.bHTLLl.rst index b181977d8d195c..009ba11ae16683 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-10-29-12-33-33.gh-issue-111438.bHTLLl.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-29-12-33-33.gh-issue-111438.bHTLLl.rst @@ -1 +1,2 @@ -Added support for sharing of float type with interpreters API. +Add support for sharing floats between interpreters using the cross-interpreter +API. Patch by Anthony Shaw. From 97f281c9a9dc8294aa6d4f891a42fb7740b53384 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sun, 5 Nov 2023 13:18:45 +1100 Subject: [PATCH 25/29] Add a recursion check --- Python/crossinterp.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 1c0da644ed9206..3a95a989a0eadf 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -781,11 +781,14 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, return -1; // PyErr_NoMemory already set } PyObject *item = PyTuple_GET_ITEM(obj, i); - if (_PyObject_GetCrossInterpreterData(item, shared->data[i]) != 0) { + + if (_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple") || + _PyObject_GetCrossInterpreterData(item, shared->data[i]) != 0) { PyMem_RawFree(shared->data[i]); _tuple_shared_free(shared); return -1; } + _Py_LeaveRecursiveCallTstate(tstate); } data->free = _tuple_shared_free; return 0; From f4ee75fb0672e6c456a06c894f739ab76b03deed Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 7 Nov 2023 07:46:27 +1100 Subject: [PATCH 26/29] Update Python/crossinterp.c Co-authored-by: Eric Snow --- Python/crossinterp.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 9e754db4f8af12..5f1205ea1522b6 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -775,23 +775,29 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, } for (Py_ssize_t i = 0; i < shared->len; i++) { - shared->data[i] = _PyCrossInterpreterData_New(); - if (shared->data[i] == NULL){ - _tuple_shared_free(shared); - return -1; // PyErr_NoMemory already set + _PyCrossInterpreterData *data = _PyCrossInterpreterData_New(); + if (data == NULL) { + goto error; // PyErr_NoMemory already set } PyObject *item = PyTuple_GET_ITEM(obj, i); - if (_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple") || - _PyObject_GetCrossInterpreterData(item, shared->data[i]) != 0) { - PyMem_RawFree(shared->data[i]); - _tuple_shared_free(shared); - return -1; + int res = -1; + if (!_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple") { + res = _PyObject_GetCrossInterpreterData(item, data); + _Py_LeaveRecursiveCallTstate(tstate); + } + if (res < 0) { + PyMem_RawFree(data); + goto error; } - _Py_LeaveRecursiveCallTstate(tstate); + shared->data[i] = data; } data->free = _tuple_shared_free; return 0; + +error: + _tuple_shared_free(shared); + return -1; } static void From 1e1141e4b9d5e01c32f26070adbb04089ebf1eef Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 7 Nov 2023 07:46:48 +1100 Subject: [PATCH 27/29] Update Python/crossinterp.c Co-authored-by: Eric Snow --- Python/crossinterp.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 5f1205ea1522b6..aeb8373bc3e80a 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -742,9 +742,15 @@ static void _tuple_shared_free(void* data) { struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data); +#ifndef NDEBUG + int64_t interpid = PyInterpreterState_GetID(_PyInterpreterState_GET())); +#endif for (Py_ssize_t i = 0; i < shared->len; i++) { if (shared->data[i] != NULL) { + assert(shared->data[i]->interpid == interpid); _PyCrossInterpreterData_Release(shared->data[i]); + PyMem_RawFree(shared->data[i]); + shared->data[i] = NULL; } } PyMem_Free(shared->data); From 1ea81a3ed336b0964dfcf45668937f2991c36adc Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 7 Nov 2023 07:56:26 +1100 Subject: [PATCH 28/29] Fix parens mismatch --- Python/crossinterp.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index aeb8373bc3e80a..a6c2b7210baedb 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -743,7 +743,7 @@ _tuple_shared_free(void* data) { struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data); #ifndef NDEBUG - int64_t interpid = PyInterpreterState_GetID(_PyInterpreterState_GET())); + int64_t interpid = PyInterpreterState_GetID(_PyInterpreterState_GET()); #endif for (Py_ssize_t i = 0; i < shared->len; i++) { if (shared->data[i] != NULL) { @@ -788,7 +788,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, PyObject *item = PyTuple_GET_ITEM(obj, i); int res = -1; - if (!_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple") { + if (!_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple")) { res = _PyObject_GetCrossInterpreterData(item, data); _Py_LeaveRecursiveCallTstate(tstate); } @@ -800,7 +800,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, } data->free = _tuple_shared_free; return 0; - + error: _tuple_shared_free(shared); return -1; From a18939a88cbf949a1e161b942ee12f18cbf29d2e Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 7 Nov 2023 09:28:15 +1100 Subject: [PATCH 29/29] Refactor to avoid the failure case running PyMem_RawFree on already freed memory space --- Python/crossinterp.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index a6c2b7210baedb..c0012aee26ea4d 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -765,14 +765,12 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, if (len < 0) { return -1; } - if (_PyCrossInterpreterData_InitWithSize( - data, tstate->interp, sizeof(struct _shared_tuple_data), obj, - _new_tuple_object - ) < 0) - { + struct _shared_tuple_data *shared = PyMem_RawMalloc(sizeof(struct _shared_tuple_data)); + if (shared == NULL){ + PyErr_NoMemory(); return -1; } - struct _shared_tuple_data *shared = (struct _shared_tuple_data *)data->data; + shared->len = len; shared->data = (_PyCrossInterpreterData **) PyMem_Calloc(shared->len, sizeof(_PyCrossInterpreterData *)); if (shared->data == NULL) { @@ -798,6 +796,8 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, } shared->data[i] = data; } + _PyCrossInterpreterData_Init( + data, tstate->interp, shared, obj, _new_tuple_object); data->free = _tuple_shared_free; return 0;