8000 Implement Immortal Instances · python/cpython@0c930b7 · GitHub
[go: up one dir, main page]
Skip to content

Commit 0c930b7

Browse files
eduardo-elizondoeduardo-elizondo
committed
Implement Immortal Instances
1 parent 4b222c9 commit 0c930b7

File tree

4 files changed

+225
-3
lines changed

4 files changed

+225
-3
lines changed

‎Include/object.h

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,12 +124,45 @@ typedef struct {
124124
#define Py_TYPE(ob) (_PyObject_CAST(ob)->ob_type)
125125
#define Py_SIZE(ob) (_PyVarObject_CAST(ob)->ob_size)
126126

127+
128+
/* [RFC] Should we enable Immortal Instances by Default? */
129+
#define Py_IMMORTAL_INSTANCES
130+
131+
/* Immortalizing causes the instance to not participate in reference counting.
132+
* Thus, an immortal object will be kept alive until the runtime finalization.
133+
* This avoids an unnecessary copy-on-write for applications that share
134+
* a common python heap across many processes. */
135+
#ifdef Py_IMMORTAL_INSTANCES
136+
137+
/* The GC bit-shifts refcounts left by two, and after that shift we still
138+
* need this to be >> 0, so leave three high zero bits (the sign bit and
139+
* room for a shift of two.) */
140+
static const Py_ssize_t kImmortalBit = 1L << (8 * sizeof(Py_ssize_t) - 4);
141+
142+
static inline int _Py_IsImmortal(PyObject *op)
143+
{
144+
return (op->ob_refcnt & kImmortalBit) != 0;
145+
}
146+
147+
static inline void _Py_SetImmortal(PyObject *op)
148+
{
149+
op->ob_refcnt = kImmortalBit;
150+
}
151+
152+
#endif /* Py_IMMORTAL_INSTANCES */
153+
154+
127155
static inline int _Py_IS_TYPE(const PyObject *ob, const PyTypeObject *type) {
128156
return ob->ob_type == type;
129157
}
130158
#define Py_IS_TYPE(ob, type) _Py_IS_TYPE(_PyObject_CAST_CONST(ob), type)
131159

132160
static inline void _Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) {
161+
#ifdef Py_IMMORTAL_INSTANCES
162+
if (_Py_IsImmortal(ob)) {
163+
return;
164+
}
165+
#endif /* Py_IMMORTAL_INSTANCES */
133166
ob->ob_refcnt = refcnt;
134167
}
135168
#define Py_SET_REFCNT(ob, refcnt) _Py_SET_REFCNT(_PyObject_CAST(ob), refcnt)
@@ -358,7 +391,6 @@ given type object has a specified feature.
358391
/* Type structure has tp_finalize member (3.4) */
359392
#define Py_TPFLAGS_HAVE_FINALIZE (1UL << 0)
360393

361-
362394
/*
363395
The macros Py_INCREF(op) and Py_DECREF(op) are used to increment or decrement
364396
reference counts. Py_DECREF calls the object's deallocator function when
@@ -397,6 +429,11 @@ PyAPI_FUNC(void) _Py_Dealloc(PyObject *);
397429

398430
static inline void _Py_INCREF(PyObject *op)
399431
{
432+
#ifdef Py_IMMORTAL_INSTANCES
433+
if (_Py_IsImmortal(op)) {
434+
return;
435+
}
436+
#endif /* Py_IMMORTAL_INSTANCES */
400437
#ifdef Py_REF_DEBUG
401438
_Py_RefTotal++;
402439
#endif
@@ -411,6 +448,11 @@ static inline void _Py_DECREF(
411448
#endif
412449
PyObject *op)
413450
{
451+
#ifdef Py_IMMORTAL_INSTANCES
452+
if (_Py_IsImmortal(op)) {
453+
return;
454+
}
455+
#endif /* Py_IMMORTAL_INSTANCES */
414456
#ifdef Py_REF_DEBUG
415457
_Py_RefTotal--;
416458
#endif

‎Lib/test/test_gc.py

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1039,6 +1039,46 @@ class Z:
10391039
gc.enable()
10401040

10411041

1042+
# These tests need to be run in a separate process since gc.immortalize_heap
1043+
# will mess up with the reference count of other tests
1044+
class GCImmortalizeTests(unittest.TestCase):
1045+
def test_not_immortal(self):
1046+
obj = []
1047+
self.assertFalse(gc.is_immortal(obj))
1048+
1049+
def test_is_immortal(self):
1050+
code = """if 1:
1051+
import gc
1052+
obj = []
1053+
gc.immortalize_heap()
1054+
print(gc.is_immortal(obj))
1055+
"""
1056+
rc, out, err = assert_python_ok('-c', code)
1057+< 9E7A /span>
self.assertEqual(out.strip(), b'True')
1058+
1059+
def test_post_immortalize(self):
1060+
code = """if 1:
1061+
import gc
1062+
gc.immortalize_heap()
1063+
obj = []
1064+
print(gc.is_immortal(obj))
1065+
"""
1066+
rc, out, err = assert_python_ok('-c', code)
1067+
self.assertEqual(out.strip(), b'False')
1068+
1069+
def test_become_tracked_after_immortalize(self):
1070+
code = """if 1:
1071+
import gc
1072+
d = {} # untracked by gc
1073+
gc.immortalize_heap()
1074+
d["foo"] = [] # now becomes gc-tracked
1075+
gc.collect() # gc should not collect immortal objects
1076+
print(len(d))
1077+
"""
1078+
rc, out, err = assert_python_ok('-c', code)
1079+
self.assertEqual(out.strip(), b'1')
1080+
1081+
10421082
class GCCallbackTests(unittest.TestCase):
10431083
def setUp(self):
10441084
# Save gc state and disable it.
@@ -1369,7 +1409,8 @@ def test_main():
13691409

13701410
try:
13711411
gc.collect() # Delete 2nd generation garbage
1372-
run_unittest(GCTests, GCTogglingTests, GCCallbackTests)
1412+
run_unittest(
1413+
GCTests, GCTogglingTests, GCCallbackTests, GCImmortalizeTests)
13731414
finally:
13741415
gc.set_debug(debug)
13751416
# test gc.enable() even if GC is disabled by default

‎Modules/clinic/gcmodule.c.h

Lines changed: 59 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Modules/gcmodule.c

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1939,6 +1939,79 @@ gc_get_freeze_count_impl(PyObject *module)
19391939
return gc_list_size(&gcstate->permanent_generation.head);
19401940
}
19411941

1942+
/*[clinic input]
1943+
gc.is_immortal -> bool
1944+
1945+
instance: object
1946+
The instance to perform the immortal check on.
1947+
1948+
Check if an object has been immortalized.
1949+
[clinic start generated code]*/
1950+
1951+
static int
1952+
gc_is_immortal_impl(PyObject *module, PyObject *instance)
1953+
/*[clinic end generated code: output=743a163cb6aaf384 input=815182ca5c5d81e4]*/
1954+
{
1955+
return _Py_IsImmortal(instance);
1956+
}
1957+
1958+
1959+
static int
1960+
immortalize_object(PyObject *obj, PyObject *Py_UNUSED(ignored))
1961+
{
1962+
_Py_SetImmortal(obj);
1963+
/* Special case for PyCodeObjects since they don't have a tp_traverse */
1964+
if (PyCode_Check(obj)) {
1965+
PyCodeObject *code = (PyCodeObject *)obj;
1966+
_Py_SetImmortal(code->co_code);
1967+
_Py_SetImmortal(code->co_consts);
1968+
_Py_SetImmortal(code->co_names);
1969+
_Py_SetImmortal(code->co_varnames);
1970+
_Py_SetImmortal(code->co_freevars);
1971+
_Py_SetImmortal(code->co_cellvars);
1972+
_Py_SetImmortal(code->co_filename);
1973+
_Py_SetImmortal(code->co_name);
1974+
_Py_SetImmortal(code->co_lnotab);
1975+
}
1976+
return 0;
1977+
}
1978+
1979+
/*[clinic input]
1980+
gc.immortalize_heap
1981+
1982+
Immortalize all instances accessible through the GC roots.
1983+
[clinic start generated code]*/
1984+
1985+
static PyObject *
1986+
gc_immortalize_heap_impl(PyObject *module)
1987+
/*[clinic end generated code: output=a7bb85fe2e27e4ae input=ca1709e4667c0623]*/
1988+
{
1989+
PyGC_Head *gc, *list;
1990+
PyThreadState *tstate = _PyThreadState_GET();
1991+
GCState *gcstate = &tstate->interp->gc;
1992+
1993+
/* Remove any dead objects to avoid immortalizing them */
1994+
PyGC_Collect();
1995+
1996+
/* Move all instances into the permanent generation */
1997+
gc_freeze_impl(module);
1998+
1999+
/* Immortalize all instances in the permanent generation */
2000+
list = &gcstate->permanent_generation.head;
2001+
for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) {
2002+
_Py_SetImmortal(FROM_GC(gc));
2003+
/* This can traverse to non-GC-tracked objects, and some of those
2004+
* non-GC-tracked objects (e.g. dicts) can later become GC-tracked, and
2005+
* not be in the permanent generation. So it is possible for immortal
2006+
* objects to enter GC collection. Currently what happens in that case
2007+
* is that their immortal bit makes it look like they have a very large
2008+
* refcount, so they are not collected.
2009+
*/
2010+
Py_TYPE(FROM_GC(gc))->tp_traverse(
2011+
FROM_GC(gc), (visitproc)immortalize_object, NULL);
2012+
}
2013+
Py_RETURN_NONE;
2014+
}
19422015

19432016
PyDoc_STRVAR(gc__doc__,
19442017
"This module provides access to the garbage collector for reference cycles.\n"
@@ -1958,6 +2031,10 @@ PyDoc_STRVAR(gc__doc__,
19582031
"is_finalized() -- Returns true if a given object has been already finalized.\n"
19592032
"get_referrers() -- Return the list of objects that refer to an object.\n"
19602033
"get_referents() -- Return the list of objects that an object refers to.\n"
2034+
#ifdef Py_IMMORTAL_INSTANCES
2035+
"immortalize_heap() -- Immortalize all instances accessible through the GC roots.\n"
2036+
"is_immortal() -- Check if an object has been immortalized.\n"
2037+
#endif
19612038
"freeze() -- Freeze all tracked objects and ignore them for future collections.\n"
19622039
"unfreeze() -- Unfreeze all objects in the permanent generation.\n"
19632040
"get_freeze_count() -- Return the number of objects in the permanent generation.\n");
@@ -1983,6 +2060,10 @@ static PyMethodDef GcMethods[] = {
19832060
GC_FREEZE_METHODDEF
19842061
GC_UNFREEZE_METHODDEF
19852062
GC_GET_FREEZE_COUNT_METHODDEF
2063+
#ifdef Py_IMMORTAL_INSTANCES
2064+
GC_IMMORTALIZE_HEAP_METHODDEF
2065+
GC_IS_IMMORTAL_METHODDEF
2066+
#endif
19862067
{NULL, NULL} /* Sentinel */
19872068
};
19882069

0 commit comments

Comments
 (0)
0