10000 gh-112075: use per-thread dict version pool (#118676) · python/cpython@ff6cbb2 · GitHub
[go: up one dir, main page]
Skip to content

Commit ff6cbb2

Browse files
DinoVDinoV
authored
gh-112075: use per-thread dict version pool (#118676)
use thread state set of dict versions
1 parent 723d4d2 commit ff6cbb2

File tree

5 files changed

+70
-3
lines changed

5 files changed

+70
-3
lines changed

‎Include/cpython/pystate.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ struct _ts {
188188

189189
PyObject *previous_executor;
190190

191+
uint64_t dict_global_version;
191192
};
192193

193194
#ifdef Py_DEBUG

‎Include/internal/pycore_dict.h

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -221,8 +221,25 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
221221
#define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1)
222222

223223
#ifdef Py_GIL_DISABLED
224-
#define DICT_NEXT_VERSION(INTERP) \
225-
(_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT)
224+
225+
#define THREAD_LOCAL_DICT_VERSION_COUNT 256
226+
#define THREAD_LOCAL_DICT_VERSION_BATCH THREAD_LOCAL_DICT_VERSION_COUNT * DICT_VERSION_INCREMENT
227+
228+
static inline uint64_t
229+
dict_next_version(PyInterpreterState *interp)
230+
{
231+
PyThreadState *tstate = PyThreadState_GET();
232+
uint64_t cur_progress = (tstate->dict_global_version &
233+
(THREAD_LOCAL_DICT_VERSION_BATCH - 1));
234+
if (cur_progress == 0) {
235+
uint64_t next = _Py_atomic_add_uint64(&interp->dict_state.global_version,
236+
THREAD_LOCAL_DICT_VERSION_BATCH);
237+
tstate->dict_global_version = next;
238+
}
239+
return tstate->dict_global_version += DICT_VERSION_INCREMENT;
240+
}
241+
242+
#define DICT_NEXT_VERSION(INTERP) dict_next_version(INTERP)
226243

227244
#else
228245
#define DICT_NEXT_VERSION(INTERP) \

‎Lib/test/test_free_threading/test_dict.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
from threading import Thread
99
from unittest import TestCase
1010

11+
from _testcapi import dict_version
12+
1113
from test.support import threading_helper
1214

1315

@@ -137,5 +139,39 @@ def writer_func(l):
137139
for ref in thread_list:
138140
self.assertIsNone(ref())
139141

142+
def test_dict_version(self):
143+
THREAD_COUNT = 10
144+
DICT_COUNT = 10000
145+
lists = []
146+
writers = []
147+
148+
def writer_func(thread_list):
149+
for i in range(DICT_COUNT):
150+
thread_list.append(dict_version({}))
151+
152+
for x in range(THREAD_COUNT):
153+
thread_list = []
154+
lists.append(thread_list)
155+
writer = Thread(target=partial(writer_func, thread_list))
156+
writers.append(writer)
157+
158+
for writer in writers:
159+
writer.start()
160+
161+
for writer in writers:
162+
writer.join()
163+
164+
total_len = 0
165+
values = set()
166+
for thread_list in lists:
167+
for v in thread_list:
168+
if v in values:
169+
print('dup', v, (v/4096)%256)
170+
values.add(v)
171+
total_len += len(thread_list)
172+
versions = set(dict_version for thread_list in lists for dict_version in thread_list)
173+
self.assertEqual(len(versions), THREAD_COUNT*DICT_COUNT)
174+
175+
140176
if __name__ == "__main__":
141177
unittest.main()

‎Modules/_testcapi/dict.c

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
#include "parts.h"
22
#include "util.h"
33

4-
54
static PyObject *
65
dict_containsstring(PyObject *self, PyObject *args)
76
{
@@ -182,6 +181,18 @@ dict_popstring_null(PyObject *self, PyObject *args)
182181
RETURN_INT(PyDict_PopString(dict, key, NULL));
183182
}
184183

184+
static PyObject *
185+
dict_version(PyObject *self, PyObject *dict)
186+
{
187+
if (!PyDict_Check(dict)) {
188+
PyErr_SetString(PyExc_TypeError, "expected dict");
189+
return NULL;
190+
}
191+
_Py_COMP_DIAG_PUSH
192+
_Py_COMP_DIAG_IGNORE_DEPR_DECLS
193+
return PyLong_FromUnsignedLongLong(((PyDictObject *)dict)->ma_version_tag);
194+
_Py_COMP_DIAG_POP
195+
}
185196

186197
static PyMethodDef test_methods[] = {
187198
{"dict_containsstring", dict_containsstring, METH_VARARGS},
@@ -193,6 +204,7 @@ static PyMethodDef test_methods[] = {
193204
{"dict_pop_null", dict_pop_null, METH_VARARGS},
194205
{"dict_popstring", dict_popstring, METH_VARARGS},
195206
{"dict_popstring_null", dict_popstring_null, METH_VARARGS},
207+
{"dict_version", dict_version, METH_O},
196208
{NULL},
197209
};
198210

‎Python/pystate.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1488,6 +1488,7 @@ init_threadstate(_PyThreadStateImpl *_tstate,
14881488
tstate->datastack_limit = NULL;
14891489
tstate->what_event = -1;
14901490
tstate->previous_executor = NULL;
1491+
tstate->dict_global_version = 0;
14911492

14921493
tstate->delete_later = NULL;
14931494

0 commit comments

Comments
 (0)
0