8000 Merge remote-tracking branch 'upstream/main' into instrumented · python/cpython@c12c317 · GitHub
[go: up one dir, main page]
Skip to content

Commit c12c317

Browse files
iritkatrieliritkatriel
committed
Merge remote-tracking branch 'upstream/main' into instrumented
2 parents 02f7a9f + b0202a4 commit c12c317

File tree

8 files changed

+255
-106
lines changed

8 files changed

+255
-106
lines changed

‎Doc/library/multiprocessing.shared_memory.rst

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -255,16 +255,17 @@ shared memory blocks created using that manager are all released when the
255255
:keyword:`with` statement's code block finishes execution.
256256

257257

258-
.. class:: ShareableList(sequence=None, *, name=None)
258+
.. class:: ShareableList(sequence=None, \*, name=None)
259259

260260
Provides a mutable list-like object where all values stored within are
261261
stored in a shared memory block. This constrains storable values to
262-
only the ``int``, ``float``, ``bool``, ``str`` (less than 10M bytes each),
263-
``bytes`` (less than 10M bytes each), and ``None`` built-in data types.
264-
It also notably differs from the built-in ``list`` type in that these
265-
lists can not change their overall length (i.e. no append, insert, etc.)
266-
and do not support the dynamic creation of new :class:`ShareableList`
267-
instances via slicing.
262+
only the ``int`` (signed 64-bit), ``float``, ``bool``, ``str`` (less
263+
than 10M bytes each when encoded as utf-8), ``bytes`` (less than 10M
264+
bytes each), and ``None`` built-in data types. It also notably
265+
differs from the built-in ``list`` type in that these lists can not
266+
change their overall length (i.e. no append, insert, etc.) and do not
267+
support the dynamic creation of new :class:`ShareableList` instances
268+
via slicing.
268269

269270
*sequence* is used in populating a new ``ShareableList`` full of values.
270271
Set to ``None`` to instead attach to an already existing
@@ -275,6 +276,35 @@ shared memory blocks created using that manager are all released when the
275276
existing ``ShareableList``, specify its shared memory block's unique
276277
name while leaving ``sequence`` set to ``None``.
277278

279+
.. note::
280+
281+
A known issue exists for :class:`bytes` and :class:`str` values.
282+
If they end with ``\x00`` nul bytes or characters, those may be
283+
*silently stripped* when fetching them by index from the
284+
:class:`ShareableList`. This ``.rstrip(b'\x00')`` behavior is
285+
considered a bug and may go away in the future. See :gh:`106939`.
286+
287+
For applications where rstripping of trailing nulls is a problem,
288+
work around it by always unconditionally appending an extra non-0
289+
byte to the end of such values when storing and unconditionally
290+
removing it when fetching:
291+
292+
.. doctest::
293+
294+
>>> from multiprocessing import shared_memory
295+
>>> nul_bug_demo = shared_memory.ShareableList(['?\x00', b'\x03\x02\x01\x00\x00\x00'])
296+
>>> nul_bug_demo[0]
297+
'?'
298+
>>> nul_bug_demo[1]
299+
b'\x03\x02\x01'
300+
>>> nul_bug_demo.shm.unlink()
301+
>>> padded = shared_memory.ShareableList(['?\x00\x07', b'\x03\x02\x01\x00\x00\x00\x07'])
302+
>>> padded[0][:-1]
303+
'?\x00'
304+
>>> padded[1][:-1]
305+
b'\x03\x02\x01\x00\x00\x00'
306+
>>> padded.shm.unlink()
307+
278308
.. method:: count(value)
279309

280310
Returns the number of occurrences of ``value``.

‎Include/internal/pycore_flowgraph.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,8 @@ int _PyCfgBuilder_Init(_PyCfgBuilder *g);
8989
void _PyCfgBuilder_Fini(_PyCfgBuilder *g);
9090

9191
int _PyCfg_OptimizeCodeUnit(_PyCfgBuilder *g, PyObject *consts, PyObject *const_cache,
92-
int code_flags, int nlocals, int nparams, int firstlineno);
93-
int _PyCfg_Stackdepth(_PyCfgBasicblock *entryblock, int code_flags);
92+
int nlocals, int nparams, int firstlineno);
93+
int _PyCfg_Stackdepth(_PyCfgBuilder *g);
9494
void _PyCfg_ConvertPseudoOps(_PyCfgBasicblock *entryblock);
9595
int _PyCfg_ResolveJumps(_PyCfgBuilder *g);
9696

‎Lib/test/test_clinic.py

Lines changed: 75 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1348,18 +1348,18 @@ def test_scaffolding(self):
13481348

13491349
class ClinicExternalTest(TestCase):
13501350
maxDiff = None
1351+
clinic_py = os.path.join(test_tools.toolsdir, "clinic", "clinic.py")
13511352

13521353
def _do_test(self, *args, expect_success=True):
1353-
clinic_py = os.path.join(test_tools.toolsdir, "clinic", "clinic.py")
13541354
with subprocess.Popen(
1355-
[sys.executable, "-Xutf8", clinic_py, *args],
1355+
[sys.executable, "-Xutf8", self.clinic_py, *args],
13561356
encoding="utf-8",
13571357
bufsize=0,
13581358
stdout=subprocess.PIPE,
13591359
stderr=subprocess.PIPE,
13601360
) as proc:
13611361
proc.wait()
1362-
if expect_success == bool(proc.returncode):
1362+
if expect_success and proc.returncode:
13631363
self.fail("".join(proc.stderr))
13641364
stdout = proc.stdout.read()
13651365
stderr = proc.stderr.read()
@@ -1449,6 +1449,49 @@ def test_cli_force(self):
14491449
generated = f.read()
14501450
self.assertTrue(generated.endswith(checksum))
14511451

1452+
def test_cli_make(self):
1453+
c_code = dedent("""
1454+
/*[clinic input]
1455+
[clinic start generated code]*/
1456+
""")
1457+
py_code = "pass"
1458+
c_files = "file1.c", "file2.c"
1459+
py_files = "file1.py", "file2.py"
1460+
1461+
def create_files(files, srcdir, code):
1462+
for fn in files:
1463+
path = os.path.join(srcdir, fn)
1464+
with open(path, "w", encoding="utf-8") as f:
1465+
f.write(code)
1466+
1467+
with os_helper.temp_dir() as tmp_dir:
1468+
# add some folders, some C files and a Python file
1469+
create_files(c_files, tmp_dir, c_code)
1470+
create_files(py_files, tmp_dir, py_code)
1471+
1472+
# create C files in externals/ dir
1473+
ext_path = os.path.join(tmp_dir, "externals")
1474+
with os_helper.temp_dir(path=ext_path) as externals:
1475+
create_files(c_files, externals, c_code)
1476+
1477+
# run clinic in verbose mode with --make on tmpdir
1478+
out = self.expect_success("-v", "--make", "--srcdir", tmp_dir)
1479+
1480+
# expect verbose mode to only mention the C files in tmp_dir
1481+
for filename in c_files:
1482+
with self.subTest(filename=filename):
1483+
path = os.path.join(tmp_dir, filename)
1484+
self.assertIn(path, out)
1485+
for filename in py_files:
1486+
with self.subTest(filename=filename):
1487+
path = os.path.join(tmp_dir, filename)
1488+
self.assertNotIn(path, out)
1489+
# don't expect C files from the externals dir
1490+
for filename in c_files:
1491+
with self.subTest(filename=filename):
1492+
path = os.path.join(ext_path, filename)
1493+
self.assertNotIn(path, out)
1494+
14521495
def test_cli_verbose(self):
14531496
with os_helper.temp_dir() as tmp_dir:
14541497
fn = os.path.join(tmp_dir, "test.c")
@@ -1534,6 +1577,35 @@ def test_cli_converters(self):
15341577
f"expected converter {converter!r}, got {line!r}"
15351578
)
15361579

1580+
def test_cli_fail_converters_and_filename(self):
1581+
out = self.expect_failure("--converters", "test.c")
1582+
msg = (
1583+
"Usage error: can't specify --converters "
1584+
"and a filename at the same time"
1585+
)
1586+
self.assertIn(msg, out)
1587+
1588+
def test_cli_fail_no_filename(self):
1589+
out = self.expect_failure()
1590+
self.assertIn("usage: clinic.py", out)
1591+
1592+
def test_cli_fail_output_and_multiple_files(self):
1593+
out = self.expect_failure("-o", "out.c", "input.c", "moreinput.c")
1594+
msg = "Usage error: can't use -o with multiple filenames"
1595+
self.assertIn(msg, out)
1596+
1597+
def test_cli_fail_filename_or_output_and_make(self):
1598+
for opts in ("-o", "out.c"), ("filename.c",):
1599+
with self.subTest(opts=opts):
1600+
out = self.expect_failure("--make", *opts)
1601+
msg = "Usage error: can't use -o or filenames with --make"
1602+
self.assertIn(msg, out)
1603+
1604+
def test_cli_fail_make_without_srcdir(self):
1605+
out = self.expect_failure("--make", "--srcdir", "")
1606+
msg = "Usage error: --srcdir must not be empty with --make"
1607+
self.assertIn(msg, out)
1608+
15371609

15381610
try:
15391611
import _testclinic as ac_tester

‎Lib/test/test_peepholer.py

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -991,14 +991,15 @@ def test_conditional_jump_forward_non_const_condition(self):
991991
('LOAD_NAME', 1, 11),
992992
('POP_JUMP_IF_TRUE', lbl := self.Label(), 12),
993993
('LOAD_CONST', 2, 13),
994+
('RETURN_VALUE', 13),
994995
lbl,
995996
('LOAD_CONST', 3, 14),
996997
('RETURN_VALUE', 14),
997998
]
998999
expected_insts = [
9991000
('LOAD_NAME', 1, 11),
10001001
('POP_JUMP_IF_TRUE', lbl := self.Label(), 12),
1001-
('LOAD_CONST', 1, 13),
1002+
('RETURN_CONST', 1, 13),
10021003
lbl,
10031004
('RETURN_CONST', 2, 14),
10041005
]
@@ -1072,6 +1073,7 @@ def test_no_unsafe_static_swap(self):
10721073
('STORE_FAST', 1, 4),
10731074
('STORE_FAST', 1, 4),
10741075
('POP_TOP', 0, 4),
1076+
('LOAD_CONST', 0, 5),
10751077
('RETURN_VALUE', 5)
10761078
]
10771079
expected_insts = [
@@ -1080,7 +1082,7 @@ def test_no_unsafe_static_swap(self):
10801082
('NOP', 0, 3),
10811083
('STORE_FAST', 1, 4),
10821084
('POP_TOP', 0, 4),
1083-
('RETURN_VALUE', 5)
1085+
('RETURN_CONST', 0)
10841086
]
10851087
self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1)
10861088

@@ -1092,6 +1094,7 @@ def test_dead_store_elimination_in_same_lineno(self):
10921094
('STORE_FAST', 1, 4),
10931095
('STORE_FAST', 1, 4),
10941096
('STORE_FAST', 1, 4),
1097+
('LOAD_CONST', 0, 5),
10951098
('RETURN_VALUE', 5)
10961099
]
10971100
expected_insts = [
@@ -1100,7 +1103,7 @@ def test_dead_store_elimination_in_same_lineno(self):
11001103
('NOP', 0, 3),
11011104
('POP_TOP', 0, 4),
11021105
('STORE_FAST', 1, 4),
1103-
('RETURN_VALUE', 5)
1106+
('RETURN_CONST'< 10000 /span>, 0, 5)
11041107
]
11051108
self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1)
11061109

@@ -1112,9 +1115,19 @@ def test_no_dead_store_elimination_in_different_lineno(self):
11121115
('STORE_FAST', 1, 4),
11131116
('STORE_FAST', 1, 5),
11141117
('STORE_FAST', 1, 6),
1118+
('LOAD_CONST', 0, 5),
11151119
('RETURN_VALUE', 5)
11161120
]
1117-
self.cfg_optimization_test(insts, insts, consts=list(range(3)), nlocals=1)
1121+
expected_insts = [
1122+
('LOAD_CONST', 0, 1),
1123+
('LOAD_CONST', 1, 2),
1124+
('LOAD_CONST', 2, 3),
1125+
('STORE_FAST', 1, 4),
1126+
('STORE_FAST', 1, 5),
1127+
('STORE_FAST', 1, 6),
1128+
('RETURN_CONST', 0, 5)
1129+
]
1130+
self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1)
11181131

11191132

11201133
if __name__ == "__main__":

‎Modules/_tkinter.c

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,6 @@ typedef struct {
317317
const Tcl_ObjType *WideIntType;
318318
const Tcl_ObjType *BignumType;
319319
const Tcl_ObjType *ListType;
320-
const Tcl_ObjType *ProcBodyType;
321320
const Tcl_ObjType *StringType;
322321
} TkappObject;
323322

@@ -595,7 +594,6 @@ Tkapp_New(const char *screenName, const char *className,
595594
v->WideIntType = Tcl_GetObjType("wideInt");
596595
v->BignumType = Tcl_GetObjType("bignum");
597596
v->ListType = Tcl_GetObjType("list");
598-
v->ProcBodyType = Tcl_GetObjType("procbody");
599597
v->StringType = Tcl_GetObjType("string");
600598

601599
/* Delete the 'exit' command, which can screw things up */
@@ -1175,10 +1173,6 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value)
11751173
return result;
11761174
}
11771175

1178-
if (value->typePtr == tkapp->ProcBodyType) {
1179-
/* fall through: return tcl object. */
1180-
}
1181-
11821176
if (value->typePtr == tkapp->StringType) {
11831177
return unicodeFromTclObj(value);
11841178
}

‎Python/compile.c

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7533,14 +7533,21 @@ build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd)
75337533
return fixed;
75347534
}
75357535

7536+
#define IS_GENERATOR(CF) \
7537+
((CF) & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR))
7538+
75367539
static int
75377540
insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock,
75387541
int *fixed, int nfreevars, int code_flags)
75397542
{
75407543
assert(umd->u_firstlineno > 0);
75417544

75427545
/* Add the generator prefix instructions. */
7543-
if (code_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
7546+
if (IS_GENERATOR(code_flags)) {
7547+
/* Note that RETURN_GENERATOR + POP_TOP have a net stack effect
7548+
* of 0. This is because RETURN_GENERATOR pushes an element
7549+
* with _PyFrame_StackPush before switching stacks.
7550+
*/
75447551
cfg_instr make_gen = {
75457552
.i_opcode = RETURN_GENERATOR,
75467553
.i_oparg = 0,
@@ -7721,19 +7728,27 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache,
77217728
int nparams = (int)PyList_GET_SIZE(u->u_ste->ste_varnames);
77227729
int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames);
77237730
assert(u->u_metadata.u_firstlineno);
7724-
if (_PyCfg_OptimizeCodeUnit(&g, consts, const_cache, code_flags, nlocals,
7731+
if (_PyCfg_OptimizeCodeUnit(&g, consts, const_cache, nlocals,
77257732
nparams, u->u_metadata.u_firstlineno) < 0) {
77267733
goto error;
77277734
}
77287735

7729-
/** Assembly **/
7730-
int nlocalsplus = prepare_localsplus(&u->u_metadata, &g, code_flags);
7731-
if (nlocalsplus < 0) {
7736+
int stackdepth = _PyCfg_Stackdepth(&g);
7737+
if (stackdepth < 0) {
77327738
goto error;
77337739
}
77347740

7735-
int maxdepth = _PyCfg_Stackdepth(g.g_entryblock, code_flags);
7736-
if (maxdepth < 0) {
7741+
/* prepare_localsplus adds instructions for generators that push
7742+
* and pop an item on the stack. This assertion makes sure there
7743+
* is space on the stack for that.
7744+
* It should always be true, because at least one expression is
7745+
* required to turn a function into a generator.
7746+
*/
7747+
assert(!(IS_GENERATOR(code_flags) && stackdepth == 0));
7748+
7749+
/** Assembly **/
7750+
int nlocalsplus = prepare_localsplus(&u->u_metadata, &g, code_flags);
7751+
if (nlocalsplus < 0) {
77377752
goto error;
77387753
}
77397754

@@ -7752,7 +7767,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache,
77527767
}
77537768

77547769
co = _PyAssemble_MakeCodeObject(&u->u_metadata, const_cache, consts,
7755-
maxdepth, &optimized_instrs, nlocalsplus,
7770+
stackdepth, &optimized_instrs, nlocalsplus,
77567771
code_flags, filename);
77577772

77587773
error:
@@ -8196,8 +8211,8 @@ _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts, int nlocals)
81968211
if (instructions_to_cfg(instructions, &g) < 0) {
81978212
goto error;
81988213
}
8199-
int code_flags = 0, nparams = 0, firstlineno = 1;
8200-
if (_PyCfg_OptimizeCodeUnit(&g, consts, const_cache, code_flags, nlocals,
8214+
int nparams = 0, firstlineno = 1;
8215+
if (_PyCfg_OptimizeCodeUnit(&g, consts, const_cache, nlocals,
82018216
nparams, firstlineno) < 0) {
82028217
goto error;
82038218
}
@@ -8232,14 +8247,14 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename,
82328247
goto error;
82338248
}
82348249

8235-
int code_flags = 0;
8236-
int nlocalsplus = prepare_localsplus(umd, &g, code_flags);
8237-
if (nlocalsplus < 0) {
8250+
int stackdepth = _PyCfg_Stackdepth(&g);
8251+
if (stackdepth < 0) {
82388252
goto error;
82398253
}
82408254

8241-
int maxdepth = _PyCfg_Stackdepth(g.g_entryblock, code_flags);
8242-
if (maxdepth < 0) {
8255+
int code_flags = 0;
8256+
int nlocalsplus = prepare_localsplus(umd, &g, code_flags);
8257+
if (nlocalsplus < 0) {
82438258
goto error;
82448259
}
82458260

@@ -8262,7 +8277,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename,
82628277
goto error;
82638278
}
82648279
co = _PyAssemble_MakeCodeObject(umd, const_cache,
8265-
consts, maxdepth, &optimized_instrs,
8280+
consts, stackdepth, &optimized_instrs,
82668281
nlocalsplus, code_flags, filename);
82678282
Py_DECREF(consts);
82688283

0 commit comments

Comments
 (0)
0