8000 Merge remote-tracking branch 'upstream/main' into gh-112984 · zooba/cpython@524edf9 · GitHub
[go: up one dir, main page]
Skip to content

Commit 524edf9

Browse files
zoobazooba
committed
Merge remote-tracking branch 'upstream/main' into pythongh-112984 
2 parents b6cc6ca + 0ae60b6 commit 524edf9

File tree

18 files changed

+242
-125
lines changed

18 files changed

+242
-125
lines changed

‎Include/cpython/code.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,8 @@ struct PyCodeObject _PyCode_DEF(1);
208208
#define CO_FUTURE_GENERATOR_STOP 0x800000
209209
#define CO_FUTURE_ANNOTATIONS 0x1000000
210210

211+
#define CO_NO_MONITORING_EVENTS 0x2000000
212+
211213
/* This should be defined if a future statement modifies the syntax.
212214
For example, when a keyword is added.
213215
*/

‎Include/internal/mimalloc/mimalloc/internal.h

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -23,23 +23,6 @@ terms of the MIT license. A copy of the license can be found in the file
2323
#define mi_trace_message(...)
2424
#endif
2525

26-
#define MI_CACHE_LINE 64
27-
#if de 628C fined(_MSC_VER)
28-
#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths)
29-
#pragma warning(disable:26812) // unscoped enum warning
30-
#define mi_decl_noinline __declspec(noinline)
31-
#define mi_decl_thread __declspec(thread)
32-
#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE))
33-
#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc
34-
#define mi_decl_noinline __attribute__((noinline))
35-
#define mi_decl_thread __thread
36-
#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE)))
37-
#else
38-
#define mi_decl_noinline
39-
#define mi_decl_thread __thread // hope for the best :-)
40-
#define mi_decl_cache_align
41-
#endif
42-
4326
#if defined(__EMSCRIPTEN__) && !defined(__wasi__)
4427
#define __wasi__
4528
#endif
@@ -131,6 +114,7 @@ void _mi_segment_map_allocated_at(const mi_segment_t* segment);
131114
void _mi_segment_map_freed_at(const mi_segment_t* segment);
132115

133116
// "segment.c"
117+
extern mi_abandoned_pool_t _mi_abandoned_default; // global abandoned pool
134118
mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld);
135119
void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld);
136120
void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld);
@@ -145,7 +129,7 @@ void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, m
145129

146130
uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size); // page start for any page
147131
void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld);
148-
void _mi_abandoned_await_readers(void);
132+
void _mi_abandoned_await_readers(mi_abandoned_pool_t *pool);
149133
void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld);
150134

151135
// "page.c"

‎Include/internal/mimalloc/mimalloc/types.h

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,23 @@ terms of the MIT license. A copy of the license can be found in the file
3333
#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t)
3434
#endif
3535

36+
#define MI_CACHE_LINE 64
37+
#if defined(_MSC_VER)
38+
#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths)
39+
#pragma warning(disable:26812) // unscoped enum warning
40+
#define mi_decl_noinline __declspec(noinline)
41+
#define mi_decl_thread __declspec(thread)
42+
#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE))
43+
#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc
44+
#define mi_decl_noinline __attribute__((noinline))
45+
#define mi_decl_thread __thread
46+
#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE)))
47+
#else
48+
#define mi_decl_noinline
49+
#define mi_decl_thread __thread // hope for the best :-)
50+
#define mi_decl_cache_align
51+
#endif
52+
3653
// ------------------------------------------------------
3754
// Variants
3855
// ------------------------------------------------------
@@ -445,6 +462,28 @@ typedef struct mi_segment_s {
445462
mi_slice_t slices[MI_SLICES_PER_SEGMENT+1]; // one more for huge blocks with large alignment
446463
} mi_segment_t;
447464

465+
typedef uintptr_t mi_tagged_segment_t;
466+
467+
// Segments unowned by any thread are put in a shared pool
468+
typedef struct mi_abandoned_pool_s {
469+
// This is a list of visited abandoned pages that were full at the time.
470+
// this list migrates to `abandoned` when that becomes NULL. The use of
471+
// this list reduces contention and the rate at which segments are visited.
472+
mi_decl_cache_align _Atomic(mi_segment_t*) abandoned_visited; // = NULL
473+
474+
// The abandoned page list (tagged as it supports pop)
475+
mi_decl_cache_align _Atomic(mi_tagged_segment_t) abandoned; // = NULL
476+
477+
// Maintain these for debug purposes (these counts may be a bit off)
478+
mi_decl_cache_align _Atomic(size_t) abandoned_count;
479+
mi_decl_cache_align _Atomic(size_t) abandoned_visited_count;
480+
481+
// We also maintain a count of current readers of the abandoned list
482+
// in order to prevent resetting/decommitting segment memory if it might
483+
// still be read.
484+
mi_decl_cache_align _Atomic(size_t) abandoned_readers; // = 0
485+
} mi_abandoned_pool_t;
486+
448487

449488
// ------------------------------------------------------
450489
// Heaps
@@ -654,6 +693,7 @@ typedef struct mi_segments_tld_s {
654693
size_t peak_size; // peak size of all segments
655694
mi_stats_t* stats; // points to tld stats
656695
mi_os_tld_t* os; // points to os stats
696+
mi_abandoned_pool_t* abandoned; // pool of abandoned segments
657697
} mi_segments_tld_t;
658698

659699
// Thread local data

‎Include/internal/pycore_interp.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ extern "C" {
2727
#include "pycore_import.h" // struct _import_state
2828
#include "pycore_instruments.h" // _PY_MONITORING_EVENTS
2929
#include "pycore_list.h" // struct _Py_list_state
30+
#include "pycore_mimalloc.h" // struct _mimalloc_interp_state
3031
#include "pycore_object_state.h" // struct _py_object_state
3132
#include "pycore_obmalloc.h" // struct _obmalloc_state
3233
#include "pycore_tstate.h" // _PyThreadStateImpl
@@ -166,6 +167,10 @@ struct _is {
166167
struct _warnings_runtime_state warnings;
167168
struct atexit_state atexit;
168169

170+
#if defined(Py_GIL_DISABLED)
171+
struct _mimalloc_interp_state mimalloc;
172+
#endif
173+
169174
struct _obmalloc_state obmalloc;
170175

171176
PyObject *audit_hooks;

‎Include/internal/pycore_mimalloc.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,12 @@ typedef enum {
3535
#endif
3636

3737
#ifdef Py_GIL_DISABLED
38+
struct _mimalloc_interp_state {
39+
// When exiting, threads place any segments with live blocks in this
40+
// shared pool for other threads to claim and reuse.
41+
mi_abandoned_pool_t abandoned_pool;
42+
};
43+
3844
struct _mimalloc_thread_state {
3945
mi_heap_t *current_object_heap;
4046
mi_heap_t heaps[_Py_MIMALLOC_HEAP_COUNT];

‎Lib/pathlib/__init__.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,13 @@ def _scandir(self):
301301

302302
def _make_child_entry(self, entry):
303303
# Transform an entry yielded from _scandir() into a path object.
304-
return self._make_child_relpath(entry.name)
304+
path_str = entry.name if str(self) == '.' else entry.path
305+
path = self.with_segments(path_str)
306+
path._str = path_str
307+
path._drv = self.drive
308+
path._root = self.root
309+
path._tail_cached = self._tail + [entry.name]
310+
return path
305311

306312
def absolute(self):
307313
"""Return an absolute version of this path

‎Lib/test/test_monitoring.py

Lines changed: 55 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -750,7 +750,7 @@ class UnwindRecorder(ExceptionRecorder):
750750
event_type = E.PY_UNWIND
751751

752752
def __call__(self, code, offset, exc):
753-
self.events.append(("unwind", type(exc)))
753+
self.events.append(("unwind", type(exc), code.co_name))
754754

755755
class Exceptio 1CF5 nHandledRecorder(ExceptionRecorder):
756756

@@ -766,8 +766,27 @@ class ThrowRecorder(ExceptionRecorder):
766766
def __call__(self, code, offset, exc):
767767
self.events.append(("throw", type(exc)))
768768

769-
class ExceptionMonitoringTest(CheckEvents):
769+
class CallRecorder:
770+
771+
event_type = E.CALL
772+
773+
def __init__(self, events):
774+
self.events = events
775+
776+
def __call__(self, code, offset, func, arg):
777+
self.events.append(("call", func.__name__, arg))
778+
779+
class ReturnRecorder:
780+
781+
event_type = E.PY_RETURN
782+
783+
def __init__(self, events):
784+
self.events = events
770785

786+
def __call__(self, code, offset, val):
787+
self.events.append(("return", code.co_name, val))
788+
789+
class ExceptionMonitoringTest(CheckEvents):
771790

772791
exception_recorders = (
773792
ExceptionRecorder,
@@ -936,26 +955,48 @@ def func():
936955
)
937956
self.assertEqual(events[0], ("throw", IndexError))
938957

939-
class LineRecorder:
958+
def test_no_unwind_for_shim_frame(self):
940959

941-
event_type = E.LINE
960+
class B:
961+
def __init__(self):
962+
raise ValueError()
963+
964+
def f():
965+
try:
966+
return B()
967+
except ValueError:
968+
pass
942969

970+
for _ in range(100):
971+
f()
972+
recorders = (
973+
ReturnRecorder,
974+
UnwindRecorder
975+
)
976+
events = self.get_events(f, TEST_TOOL, recorders)
977+
adaptive_insts = dis.get_instructions(f, adaptive=True)
978+
self.assertIn(
979+
"CALL_ALLOC_AND_ENTER_INIT",
980+
[i.opname for i in adaptive_insts]
981+
)
982+
#There should be only one unwind event
983+
expected = [
984+
('unwind', ValueError, '__init__'),
985+
('return', 'f', None),
986+
]
943987

944-
def __init__(self, events):
945-
self.events = events
988+
self.assertEqual(events, expected)
946989

947-
def __call__(self, code, line):
948-
self.events.append(("line", code.co_name, line - code.co_firstlineno))
990+
class LineRecorder:
949991

950-
class CallRecorder:
992+
event_type = E.LINE
951993

952-
event_type = E.CALL
953994

954995
def __init__(self, events):
955996
self.events = events
956997

957-
def __call__(self, code, offset, func, arg):
958-
self.events.append(("call", func.__name__, arg))
998+
def __call__(self, code, line):
999+
self.events.append(("line", code.co_name, line - code.co_firstlineno))
9591000

9601001
class CEventRecorder:
9611002

@@ -1351,15 +1392,6 @@ class BranchRecorder(JumpRecorder):
13511392
event_type = E.BRANCH
13521393
name = "branch"
13531394

1354-
class ReturnRecorder:
1355-
1356-
event_type = E.PY_RETURN
1357-
1358-
def __init__(self, events):
1359-
self.events = events
1360-
1361-
def __call__(self, code, offset, val):
1362-
self.events.append(("return", val))
13631395

13641396

13651397
JUMP_AND_BRANCH_RECORDERS = JumpRecorder, BranchRecorder
@@ -1449,11 +1481,11 @@ def func():
14491481
('branch', 'func', 4, 4),
14501482
('line', 'func', 5),
14511483
('line', 'meth', 1),
1452-
('return', None),
1484+
('return', 'meth', None),
14531485
('jump', 'func', 5, 5),
14541486
('jump', 'func', 5, '[offset=114]'),
14551487
('branch', 'func', '[offset=120]', '[offset=124]'),
1456-
('return', None),
1488+
('return', 'func', None),
14571489
('line', 'get_events', 11)])
14581490

14591491
class TestLoadSuperAttr(CheckEvents):

‎Lib/test/test_typing.py

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3448,8 +3448,8 @@ def meth(self): pass
34483448

34493449
self.assertNotIn("__protocol_attrs__", vars(NonP))
34503450
self.assertNotIn("__protocol_attrs__", vars(NonPR))
3451-
self.assertNotIn("__callable_proto_members_only__", vars(NonP))
3452-
self.assertNotIn("__callable_proto_members_only__", vars(NonPR))
3451+
self.assertNotIn("__non_callable_proto_members__", vars(NonP))
3452+
self.assertNotIn("__non_callable_proto_members__", vars(NonPR))
34533453

34543454
self.assertEqual(get_protocol_members(P), {"x"})
34553455
self.assertEqual(get_protocol_members(PR), {"meth"})
@@ -4105,6 +4105,7 @@ def method(self) -> None: ...
41054105
self.assertNotIsInstance(42, ProtocolWithMixedMembers)
41064106

41074107
def test_protocol_issubclass_error_message(self):
4108+
@runtime_checkable
41084109
class Vec2D(Protocol):
41094110
x: float
41104111
y: float
@@ -4120,6 +4121,39 @@ def square_norm(self) -> float:
41204121
with self.assertRaisesRegex(TypeError, re.escape(expected_error_message)):
41214122
issubclass(int, Vec2D)
41224123

4124+
def test_nonruntime_protocol_interaction_with_evil_classproperty(self):
4125+
class classproperty:
4126+
def __get__(self, instance, type):
4127+
raise RuntimeError("NO")
4128+
4129+
class Commentable(Protocol):
4130+
evil = classproperty()
4131+
4132+
# recognised as a protocol attr,
4133+
# but not actually accessed by the protocol metaclass
4134+
# (which would raise RuntimeError) for non-runtime protocols.
4135+
# See gh-113320
4136+
self.assertEqual(get_protocol_members(Commentable), {"evil"})
4137+
4138+
def test_runtime_protocol_interaction_with_evil_classproperty(self):
4139+
class CustomError(Exception): pass
4140+
4141+
class classproperty:
4142+
def __get__(self, instance, type):
4143+
raise CustomError
4144+
4145+
with self.assertRaises(TypeError) as cm:
4146+
@runtime_checkable
4147+
class Commentable(Protocol):
4148+
evil = classproperty()
4149+
4150+
exc = cm.exception
4151+
self.assertEqual(
4152+
exc.args[0],
4153+
"Failed to determine whether protocol member 'evil' is a method member"
4154+
)
4155+
self.assertIs(type(exc.__cause__), CustomError)
4156+
41234157

41244158
class GenericTests(BaseTestCase):
41254159

0 commit comments

Comments
 (0)
0