diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 5f0454086d7..f96537ec056 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1012,8 +1012,6 @@ def return_genexp(): code_lines = self.get_code_lines(genexp_code) self.assertEqual(genexp_lines, code_lines) - # TODO: RUSTPYTHON; implicit return line number after async for - @unittest.expectedFailure def test_line_number_implicit_return_after_async_for(self): async def test(aseq): diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py new file mode 100644 index 00000000000..0100c7f5947 --- /dev/null +++ b/Lib/test/test_monitoring.py @@ -0,0 +1,2454 @@ +"""Test suite for the sys.monitoring.""" + +import collections +import dis +import functools +import inspect +import math +import operator +import sys +import textwrap +import types +import unittest + +import test.support +from test.support import import_helper, requires_specialization_ft, script_helper + +_testcapi = import_helper.import_module("_testcapi") +_testinternalcapi = import_helper.import_module("_testinternalcapi") + +PAIR = (0,1) + +def f1(): + pass + +def f2(): + len([]) + sys.getsizeof(0) + +def floop(): + for item in PAIR: + pass + +def gen(): + yield + yield + +def g1(): + for _ in gen(): + pass + +TEST_TOOL = 2 +TEST_TOOL2 = 3 +TEST_TOOL3 = 4 + +def nth_line(func, offset): + return func.__code__.co_firstlineno + offset + +class MonitoringBasicTest(unittest.TestCase): + + def tearDown(self): + sys.monitoring.free_tool_id(TEST_TOOL) + + def test_has_objects(self): + m = sys.monitoring + m.events + m.use_tool_id + m.clear_tool_id + m.free_tool_id + m.get_tool + m.get_events + m.set_events + m.get_local_events + m.set_local_events + m.register_callback + m.restart_events + m.DISABLE + m.MISSING + m.events.NO_EVENTS + + def test_tool(self): + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), "MonitoringTest.Tool") + sys.monitoring.set_events(TEST_TOOL, 15) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 15) + sys.monitoring.set_events(TEST_TOOL, 0) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RETURN) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RAISE) + sys.monitoring.free_tool_id(TEST_TOOL) + self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.CALL) + + def test_clear(self): + events = [] + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, lambda *args: events.append(args)) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, lambda *args: events.append(args)) + def f(): + a = 1 + sys.monitoring.set_local_events(TEST_TOOL, f.__code__, E.LINE) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + + f() + sys.monitoring.clear_tool_id(TEST_TOOL) + f() + + # the first f() should trigger a PY_START and a LINE event + # the second f() after clear_tool_id should not trigger any event + # the callback function should be cleared as well + self.assertEqual(len(events), 2) + callback = sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertIs(callback, None) + + sys.monitoring.free_tool_id(TEST_TOOL) + + events = [] + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.register_callback(TEST_TOOL, E.LINE, lambda *args: events.append(args)) + sys.monitoring.set_local_events(TEST_TOOL, f.__code__, E.LINE) + f() + sys.monitoring.free_tool_id(TEST_TOOL) + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + f() + # the first f() should trigger a LINE event, and even if we use the + # tool id immediately after freeing it, the second f() should not + # trigger any event + self.assertEqual(len(events), 1) + sys.monitoring.free_tool_id(TEST_TOOL) + + +class MonitoringTestBase: + + def setUp(self): + # Check that a previous test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL), None) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL2), None) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL3), None) + sys.monitoring.use_tool_id(TEST_TOOL, "test " + self.__class__.__name__) + sys.monitoring.use_tool_id(TEST_TOOL2, "test2 " + self.__class__.__name__) + sys.monitoring.use_tool_id(TEST_TOOL3, "test3 " + self.__class__.__name__) + + def tearDown(self): + # Check that test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + sys.monitoring.free_tool_id(TEST_TOOL) + sys.monitoring.free_tool_id(TEST_TOOL2) + sys.monitoring.free_tool_id(TEST_TOOL3) + + +class MonitoringCountTest(MonitoringTestBase, unittest.TestCase): + + def check_event_count(self, func, event, expected): + + class Counter: + def __init__(self): + self.count = 0 + def __call__(self, *args): + self.count += 1 + + counter = Counter() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + self.assertEqual(counter.count, 0) + counter.count = 0 + func() + self.assertEqual(counter.count, expected) + prev = sys.monitoring.register_callback(TEST_TOOL, event, None) + counter.count = 0 + func() + self.assertEqual(counter.count, 0) + self.assertEqual(prev, counter) + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_start_count(self): + self.check_event_count(f1, E.PY_START, 1) + + def test_resume_count(self): + self.check_event_count(g1, E.PY_RESUME, 2) + + def test_return_count(self): + self.check_event_count(f1, E.PY_RETURN, 1) + + def test_call_count(self): + self.check_event_count(f2, E.CALL, 3) + + def test_c_return_count(self): + self.check_event_count(f2, E.C_RETURN, 2) + + +E = sys.monitoring.events + +INSTRUMENTED_EVENTS = [ + (E.PY_START, "start"), + (E.PY_RESUME, "resume"), + (E.PY_RETURN, "return"), + (E.PY_YIELD, "yield"), + (E.JUMP, "jump"), + (E.BRANCH, "branch"), +] + +EXCEPT_EVENTS = [ + (E.RAISE, "raise"), + (E.PY_UNWIND, "unwind"), + (E.EXCEPTION_HANDLED, "exception_handled"), +] + +SIMPLE_EVENTS = INSTRUMENTED_EVENTS + EXCEPT_EVENTS + [ + (E.C_RAISE, "c_raise"), + (E.C_RETURN, "c_return"), +] + + +SIMPLE_EVENT_SET = functools.reduce(operator.or_, [ev for (ev, _) in SIMPLE_EVENTS], 0) | E.CALL + + +def just_pass(): + pass + +just_pass.events = [ + "py_call", + "start", + "return", +] + +def just_raise(): + raise Exception + +just_raise.events = [ + 'py_call', + "start", + "raise", + "unwind", +] + +def just_call(): + len([]) + +just_call.events = [ + 'py_call', + "start", + "c_call", + "c_return", + "return", +] + +def caught(): + try: + 1/0 + except Exception: + pass + +caught.events = [ + 'py_call', + "start", + "raise", + "exception_handled", + "branch", + "return", +] + +def nested_call(): + just_pass() + +nested_call.events = [ + "py_call", + "start", + "py_call", + "start", + "return", + "return", +] + +PY_CALLABLES = (types.FunctionType, types.MethodType) + +class MonitoringEventsBase(MonitoringTestBase): + + def gather_events(self, func): + events = [] + for event, event_name in SIMPLE_EVENTS: + def record(*args, event_name=event_name): + events.append(event_name) + sys.monitoring.register_callback(TEST_TOOL, event, record) + def record_call(code, offset, obj, arg): + if isinstance(obj, PY_CALLABLES): + events.append("py_call") + else: + events.append("c_call") + sys.monitoring.register_callback(TEST_TOOL, E.CALL, record_call) + sys.monitoring.set_events(TEST_TOOL, SIMPLE_EVENT_SET) + events = [] + try: + func() + except: + pass + sys.monitoring.set_events(TEST_TOOL, 0) + #Remove the final event, the call to `sys.monitoring.set_events` + events = events[:-1] + return events + + def check_events(self, func, expected=None): + events = self.gather_events(func) + if expected is None: + expected = func.events + self.assertEqual(events, expected) + +class MonitoringEventsTest(MonitoringEventsBase, unittest.TestCase): + + def test_just_pass(self): + self.check_events(just_pass) + + def test_just_raise(self): + try: + self.check_events(just_raise) + except Exception: + pass + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 0) + + def test_just_call(self): + self.check_events(just_call) + + def test_caught(self): + self.check_events(caught) + + def test_nested_call(self): + self.check_events(nested_call) + +UP_EVENTS = (E.C_RETURN, E.C_RAISE, E.PY_RETURN, E.PY_UNWIND, E.PY_YIELD) +DOWN_EVENTS = (E.PY_START, E.PY_RESUME) + +from test.profilee import testfunc + +class SimulateProfileTest(MonitoringEventsBase, unittest.TestCase): + + def test_balanced(self): + events = self.gather_events(testfunc) + c = collections.Counter(events) + self.assertEqual(c["c_call"], c["c_return"]) + self.assertEqual(c["start"], c["return"] + c["unwind"]) + self.assertEqual(c["raise"], c["exception_handled"] + c["unwind"]) + + def test_frame_stack(self): + self.maxDiff = None + stack = [] + errors = [] + seen = set() + def up(*args): + frame = sys._getframe(1) + if not stack: + errors.append("empty") + else: + expected = stack.pop() + if frame != expected: + errors.append(f" Popping {frame} expected {expected}") + def down(*args): + frame = sys._getframe(1) + stack.append(frame) + seen.add(frame.f_code) + def call(code, offset, callable, arg): + if not isinstance(callable, PY_CALLABLES): + stack.append(sys._getframe(1)) + for event in UP_EVENTS: + sys.monitoring.register_callback(TEST_TOOL, event, up) + for event in DOWN_EVENTS: + sys.monitoring.register_callback(TEST_TOOL, event, down) + sys.monitoring.register_callback(TEST_TOOL, E.CALL, call) + sys.monitoring.set_events(TEST_TOOL, SIMPLE_EVENT_SET) + testfunc() + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertEqual(errors, []) + self.assertEqual(stack, [sys._getframe()]) + self.assertEqual(len(seen), 9) + + +class CounterWithDisable: + + def __init__(self): + self.disable = False + self.count = 0 + + def __call__(self, *args): + self.count += 1 + if self.disable: + return sys.monitoring.DISABLE + + +class RecorderWithDisable: + + def __init__(self, events): + self.disable = False + self.events = events + + def __call__(self, code, event): + self.events.append(event) + if self.disable: + return sys.monitoring.DISABLE + + +class MontoringDisableAndRestartTest(MonitoringTestBase, unittest.TestCase): + + def test_disable(self): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + self.assertEqual(counter.count, 0) + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.disable = True + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + sys.monitoring.set_events(TEST_TOOL, 0) + finally: + sys.monitoring.restart_events() + + def test_restart(self): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + counter.disable = True + f1() + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + sys.monitoring.restart_events() + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + sys.monitoring.set_events(TEST_TOOL, 0) + finally: + sys.monitoring.restart_events() + + +class MultipleMonitorsTest(MonitoringTestBase, unittest.TestCase): + + def test_two_same(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2)}) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) + + def test_three_same(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + counter3 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.register_callback(TEST_TOOL3, E.PY_START, counter3) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + sys.monitoring.set_events(TEST_TOOL3, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL3), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2) | (1 << TEST_TOOL3)}) + counter1.count = 0 + counter2.count = 0 + counter3.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + count3 = counter3.count + self.assertEqual((count1, count2, count3), (1, 1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.set_events(TEST_TOOL3, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL3, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) + + def test_two_different(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_RETURN) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_RETURN) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': 1 << TEST_TOOL, 'PY_RETURN': 1 << TEST_TOOL2}) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, None) + self.assertEqual(sys.monitoring._all_events(), {}) + + def test_two_with_disable(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2)}) + counter1.count = 0 + counter2.count = 0 + counter1.disable = True + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (0, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + def test_with_instruction_event(self): + """Test that the second tool can set events with instruction events set by the first tool.""" + def f(): + pass + code = f.__code__ + + try: + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.set_local_events(TEST_TOOL, code, E.INSTRUCTION | E.LINE) + sys.monitoring.set_local_events(TEST_TOOL2, code, E.LINE) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + self.assertEqual(sys.monitoring._all_events(), {}) + + +class LineMonitoringTest(MonitoringTestBase, unittest.TestCase): + + def test_lines_single(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + sys.monitoring.set_events(TEST_TOOL, E.LINE) + f1() + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + start = nth_line(LineMonitoringTest.test_lines_single, 0) + self.assertEqual(events, [start+7, nth_line(f1, 1), start+8]) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + def test_lines_loop(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + sys.monitoring.set_events(TEST_TOOL, E.LINE) + floop() + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + start = nth_line(LineMonitoringTest.test_lines_loop, 0) + floop_1 = nth_line(floop, 1) + floop_2 = nth_line(floop, 2) + self.assertEqual( + events, + [start+7, floop_1, floop_2, floop_1, floop_2, floop_1, start+8] + ) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + def test_lines_two(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + events2 = [] + recorder2 = RecorderWithDisable(events2) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, recorder2) + sys.monitoring.set_events(TEST_TOOL, E.LINE); sys.monitoring.set_events(TEST_TOOL2, E.LINE) + f1() + sys.monitoring.set_events(TEST_TOOL, 0); sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) + start = nth_line(LineMonitoringTest.test_lines_two, 0) + expected = [start+10, nth_line(f1, 1), start+11] + self.assertEqual(events, expected) + self.assertEqual(events2, expected) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + def check_lines(self, func, expected, tool=TEST_TOOL): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(tool, E.LINE, recorder) + sys.monitoring.set_events(tool, E.LINE) + func() + sys.monitoring.set_events(tool, 0) + sys.monitoring.register_callback(tool, E.LINE, None) + lines = [ line - func.__code__.co_firstlineno for line in events[1:-1] ] + self.assertEqual(lines, expected) + finally: + sys.monitoring.set_events(tool, 0) + + + def test_linear(self): + + def func(): + line = 1 + line = 2 + line = 3 + line = 4 + line = 5 + + self.check_lines(func, [1,2,3,4,5]) + + def test_branch(self): + def func(): + if "true".startswith("t"): + line = 2 + line = 3 + else: + line = 5 + line = 6 + + self.check_lines(func, [1,2,3,6]) + + def test_try_except(self): + + def func1(): + try: + line = 2 + line = 3 + except: + line = 5 + line = 6 + + self.check_lines(func1, [1,2,3,6]) + + def func2(): + try: + line = 2 + raise 3 + except: + line = 5 + line = 6 + + self.check_lines(func2, [1,2,3,4,5,6]) + + def test_generator_with_line(self): + + def f(): + def a(): + yield + def b(): + yield from a() + next(b()) + + self.check_lines(f, [1,3,5,4,2,4]) + +class TestDisable(MonitoringTestBase, unittest.TestCase): + + def gen(self, cond): + for i in range(10): + if cond: + yield 1 + else: + yield 2 + + def raise_handle_reraise(self): + try: + 1/0 + except: + raise + + def test_disable_legal_events(self): + for event, name in INSTRUMENTED_EVENTS: + try: + counter = CounterWithDisable() + counter.disable = True + sys.monitoring.register_callback(TEST_TOOL, event, counter) + sys.monitoring.set_events(TEST_TOOL, event) + for _ in self.gen(1): + pass + self.assertLess(counter.count, 4) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, event, None) + + + def test_disable_illegal_events(self): + for event, name in EXCEPT_EVENTS: + try: + counter = CounterWithDisable() + counter.disable = True + sys.monitoring.register_callback(TEST_TOOL, event, counter) + sys.monitoring.set_events(TEST_TOOL, event) + with self.assertRaises(ValueError): + self.raise_handle_reraise() + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, event, None) + + +class ExceptionRecorder: + + event_type = E.RAISE + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, exc): + self.events.append(("raise", type(exc))) + +class CheckEvents(MonitoringTestBase, unittest.TestCase): + + def get_events(self, func, tool, recorders): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_events(tool, all_events) + func() + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + return event_list + finally: + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): + events = self.get_events(func, tool, recorders) + self.assertEqual(events, expected) + + def check_balanced(self, func, recorders): + events = self.get_events(func, TEST_TOOL, recorders) + self.assertEqual(len(events)%2, 0) + for r, h in zip(events[::2],events[1::2]): + r0 = r[0] + self.assertIn(r0, ("raise", "reraise")) + h0 = h[0] + self.assertIn(h0, ("handled", "unwind")) + self.assertEqual(r[1], h[1]) + + +class StopiterationRecorder(ExceptionRecorder): + + event_type = E.STOP_ITERATION + +class ReraiseRecorder(ExceptionRecorder): + + event_type = E.RERAISE + + def __call__(self, code, offset, exc): + self.events.append(("reraise", type(exc))) + +class UnwindRecorder(ExceptionRecorder): + + event_type = E.PY_UNWIND + + def __call__(self, code, offset, exc): + self.events.append(("unwind", type(exc), code.co_name)) + +class ExceptionHandledRecorder(ExceptionRecorder): + + event_type = E.EXCEPTION_HANDLED + + def __call__(self, code, offset, exc): + self.events.append(("handled", type(exc))) + +class ThrowRecorder(ExceptionRecorder): + + event_type = E.PY_THROW + + def __call__(self, code, offset, exc): + self.events.append(("throw", type(exc))) + +class CallRecorder: + + event_type = E.CALL + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, func, arg): + self.events.append(("call", func.__name__, arg)) + +class ReturnRecorder: + + event_type = E.PY_RETURN + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, val): + self.events.append(("return", code.co_name, val)) + + +class ExceptionMonitoringTest(CheckEvents): + + exception_recorders = ( + ExceptionRecorder, + ReraiseRecorder, + ExceptionHandledRecorder, + UnwindRecorder + ) + + def test_simple_try_except(self): + + def func1(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func1, [("raise", KeyError)]) + + def test_implicit_stop_iteration(self): + """Generators are documented as raising a StopIteration + when they terminate. + However, we don't do that if we can avoid it, for speed. + sys.monitoring handles that by injecting a STOP_ITERATION + event when we would otherwise have skip the RAISE event. + This test checks that both paths record an equivalent event. + """ + + def gen(): + yield 1 + return 2 + + def implicit_stop_iteration(iterator=None): + if iterator is None: + iterator = gen() + for _ in iterator: + pass + + recorders=(ExceptionRecorder, StopiterationRecorder,) + expected = [("raise", StopIteration)] + + # Make sure that the loop is unspecialized, and that it will not + # re-specialize immediately, so that we can we can test the + # unspecialized version of the loop first. + # Note: this assumes that we don't specialize loops over sets. + implicit_stop_iteration(set(range(_testinternalcapi.SPECIALIZATION_THRESHOLD))) + + # This will record a RAISE event for the StopIteration. + self.check_events(implicit_stop_iteration, expected, recorders=recorders) + + # Now specialize, so that we see a STOP_ITERATION event. + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): + implicit_stop_iteration() + + # This will record a STOP_ITERATION event for the StopIteration. + self.check_events(implicit_stop_iteration, expected, recorders=recorders) + + initial = [ + ("raise", ZeroDivisionError), + ("handled", ZeroDivisionError) + ] + + reraise = [ + ("reraise", ZeroDivisionError), + ("handled", ZeroDivisionError) + ] + + def test_explicit_reraise(self): + + def func(): + try: + try: + 1/0 + except: + raise + except: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + def test_explicit_reraise_named(self): + + def func(): + try: + try: + 1/0 + except Exception as ex: + raise + except: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + def test_implicit_reraise(self): + + def func(): + try: + try: + 1/0 + except ValueError: + pass + except: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + + def test_implicit_reraise_named(self): + + def func(): + try: + try: + 1/0 + except ValueError as ex: + pass + except: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + def test_try_finally(self): + + def func(): + try: + try: + 1/0 + finally: + pass + except: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + def test_async_for(self): + + def func(): + + async def async_generator(): + for i in range(1): + raise ZeroDivisionError + yield i + + async def async_loop(): + try: + async for item in async_generator(): + pass + except Exception: + pass + + try: + async_loop().send(None) + except StopIteration: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + def test_throw(self): + + def gen(): + yield 1 + yield 2 + + def func(): + try: + g = gen() + next(g) + g.throw(IndexError) + except IndexError: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + events = self.get_events( + func, + TEST_TOOL, + self.exception_recorders + (ThrowRecorder,) + ) + self.assertEqual(events[0], ("throw", IndexError)) + + @requires_specialization_ft + def test_no_unwind_for_shim_frame(self): + class ValueErrorRaiser: + def __init__(self): + raise ValueError() + + def f(): + try: + return ValueErrorRaiser() + except ValueError: + pass + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + f() + recorders = ( + ReturnRecorder, + UnwindRecorder + ) + events = self.get_events(f, TEST_TOOL, recorders) + adaptive_insts = dis.get_instructions(f, adaptive=True) + self.assertIn( + "CALL_ALLOC_AND_ENTER_INIT", + [i.opname for i in adaptive_insts] + ) + #There should be only one unwind event + expected = [ + ('unwind', ValueError, '__init__'), + ('return', 'f', None), + ] + + self.assertEqual(events, expected) + + # gh-140373 + def test_gen_unwind(self): + def gen(): + yield 1 + + def f(): + g = gen() + next(g) + g.close() + + recorders = ( + UnwindRecorder, + ) + events = self.get_events(f, TEST_TOOL, recorders) + expected = [ + ("unwind", GeneratorExit, "gen"), + ] + self.assertEqual(events, expected) + +class LineRecorder: + + event_type = E.LINE + + + def __init__(self, events): + self.events = events + + def __call__(self, code, line): + self.events.append(("line", code.co_name, line - code.co_firstlineno)) + +class CEventRecorder: + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, func, arg): + self.events.append((self.event_name, func.__name__, arg)) + +class CReturnRecorder(CEventRecorder): + + event_type = E.C_RETURN + event_name = "C return" + +class CRaiseRecorder(CEventRecorder): + + event_type = E.C_RAISE + event_name = "C raise" + +MANY_RECORDERS = ExceptionRecorder, CallRecorder, LineRecorder, CReturnRecorder, CRaiseRecorder + +class TestManyEvents(CheckEvents): + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = MANY_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('call', 'func1', sys.monitoring.MISSING), + ('line', 'func1', 1), + ('line', 'func1', 2), + ('line', 'func1', 3), + ('line', 'get_events', 11), + ('call', 'set_events', 2)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = MANY_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('call', 'func2', sys.monitoring.MISSING), + ('line', 'func2', 1), + ('line', 'func2', 2), + ('call', 'append', [2]), + ('C return', 'append', [2]), + ('line', 'func2', 3), + ('line', 'get_events', 11), + ('call', 'set_events', 2)]) + + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = MANY_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('call', 'func3', sys.monitoring.MISSING), + ('line', 'func3', 1), + ('line', 'func3', 2), + ('line', 'func3', 3), + ('raise', KeyError), + ('line', 'func3', 4), + ('line', 'func3', 5), + ('line', 'func3', 6), + ('line', 'get_events', 11), + ('call', 'set_events', 2)]) + +class InstructionRecorder: + + event_type = E.INSTRUCTION + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset): + # Filter out instructions in check_events to lower noise + if code.co_name != "get_events": + self.events.append(("instruction", code.co_name, offset)) + + +LINE_AND_INSTRUCTION_RECORDERS = InstructionRecorder, LineRecorder + +class TestLineAndInstructionEvents(CheckEvents): + maxDiff = None + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 4), + ('line', 'func1', 2), + ('instruction', 'func1', 6), + ('instruction', 'func1', 8), + ('line', 'func1', 3), + ('instruction', 'func1', 10), + ('instruction', 'func1', 12), + ('instruction', 'func1', 14), + ('instruction', 'func1', 16), + ('line', 'get_events', 11)]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - instruction offsets differ from CPython + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func2', 1), + ('instruction', 'func2', 2), + ('instruction', 'func2', 4), + ('line', 'func2', 2), + ('instruction', 'func2', 6), + ('instruction', 'func2', 8), + ('instruction', 'func2', 28), + ('instruction', 'func2', 30), + ('instruction', 'func2', 38), + ('line', 'func2', 3), + ('instruction', 'func2', 40), + ('instruction', 'func2', 42), + ('instruction', 'func2', 44), + ('instruction', 'func2', 46), + ('line', 'get_events', 11)]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - instruction offsets differ from CPython + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func3', 1), + ('instruction', 'func3', 2), + ('line', 'func3', 2), + ('instruction', 'func3', 4), + ('instruction', 'func3', 6), + ('line', 'func3', 3), + ('instruction', 'func3', 8), + ('instruction', 'func3', 18), + ('instruction', 'func3', 20), + ('line', 'func3', 4), + ('instruction', 'func3', 22), + ('line', 'func3', 5), + ('instruction', 'func3', 24), + ('instruction', 'func3', 26), + ('instruction', 'func3', 28), + ('line', 'func3', 6), + ('instruction', 'func3', 30), + ('instruction', 'func3', 32), + ('instruction', 'func3', 34), + ('instruction', 'func3', 36), + ('line', 'get_events', 11)]) + + def test_with_restart(self): + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 4), + ('line', 'func1', 2), + ('instruction', 'func1', 6), + ('instruction', 'func1', 8), + ('line', 'func1', 3), + ('instruction', 'func1', 10), + ('instruction', 'func1', 12), + ('instruction', 'func1', 14), + ('instruction', 'func1', 16), + ('line', 'get_events', 11)]) + + sys.monitoring.restart_events() + + self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 4), + ('line', 'func1', 2), + ('instruction', 'func1', 6), + ('instruction', 'func1', 8), + ('line', 'func1', 3), + ('instruction', 'func1', 10), + ('instruction', 'func1', 12), + ('instruction', 'func1', 14), + ('instruction', 'func1', 16), + ('line', 'get_events', 11)]) + + def test_turn_off_only_instruction(self): + """ + LINE events should be recorded after INSTRUCTION event is turned off + """ + events = [] + def line(*args): + events.append("line") + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, line) + sys.monitoring.register_callback(TEST_TOOL, E.INSTRUCTION, lambda *args: None) + sys.monitoring.set_events(TEST_TOOL, E.LINE | E.INSTRUCTION) + sys.monitoring.set_events(TEST_TOOL, E.LINE) + events = [] + a = 0 + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertGreater(len(events), 0) + +class TestInstallIncrementally(MonitoringTestBase, unittest.TestCase): + + def check_events(self, func, must_include, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + all_events |= recorder.event_type + sys.monitoring.set_events(tool, all_events) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, recorder(event_list)) + func() + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + for line in must_include: + self.assertIn(line, event_list) + finally: + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + @staticmethod + def func1(): + line1 = 1 + + MUST_INCLUDE_LI = [ + ('instruction', 'func1', 2), + ('line', 'func1', 2), + ('instruction', 'func1', 4), + ('instruction', 'func1', 6)] + + def test_line_then_instruction(self): + recorders = [ LineRecorder, InstructionRecorder ] + self.check_events(self.func1, + recorders = recorders, must_include = self.MUST_INCLUDE_LI) + + def test_instruction_then_line(self): + recorders = [ InstructionRecorder, LineRecorder ] + self.check_events(self.func1, + recorders = recorders, must_include = self.MUST_INCLUDE_LI) + + @staticmethod + def func2(): + len(()) + + MUST_INCLUDE_CI = [ + ('instruction', 'func2', 2), + ('call', 'func2', sys.monitoring.MISSING), + ('call', 'len', ()), + ('instruction', 'func2', 12), + ('instruction', 'func2', 14)] + + + + def test_call_then_instruction(self): + recorders = [ CallRecorder, InstructionRecorder ] + self.check_events(self.func2, + recorders = recorders, must_include = self.MUST_INCLUDE_CI) + + def test_instruction_then_call(self): + recorders = [ InstructionRecorder, CallRecorder ] + self.check_events(self.func2, + recorders = recorders, must_include = self.MUST_INCLUDE_CI) + +LOCAL_RECORDERS = CallRecorder, LineRecorder, CReturnRecorder, CRaiseRecorder + +class TestLocalEvents(MonitoringTestBase, unittest.TestCase): + + def check_events(self, func, expected, tool=TEST_TOOL, recorders=()): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_local_events(tool, func.__code__, all_events) + func() + sys.monitoring.set_local_events(tool, func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + self.assertEqual(event_list, expected) + finally: + sys.monitoring.set_local_events(tool, func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = LOCAL_RECORDERS, expected = [ + ('line', 'func1', 1), + ('line', 'func1', 2), + ('line', 'func1', 3)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = LOCAL_RECORDERS, expected = [ + ('line', 'func2', 1), + ('line', 'func2', 2), + ('call', 'append', [2]), + ('C return', 'append', [2]), + ('line', 'func2', 3)]) + + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = LOCAL_RECORDERS, expected = [ + ('line', 'func3', 1), + ('line', 'func3', 2), + ('line', 'func3', 3), + ('line', 'func3', 4), + ('line', 'func3', 5), + ('line', 'func3', 6)]) + + def test_set_non_local_event(self): + with self.assertRaises(ValueError): + sys.monitoring.set_local_events(TEST_TOOL, just_call.__code__, E.RAISE) + +def line_from_offset(code, offset): + for start, end, line in code.co_lines(): + if start <= offset < end: + if line is None: + return f"[offset={offset}]" + return line - code.co_firstlineno + return -1 + +class JumpRecorder: + + event_type = E.JUMP + name = "jump" + + def __init__(self, events): + self.events = events + + def __call__(self, code, from_, to): + from_line = line_from_offset(code, from_) + to_line = line_from_offset(code, to) + self.events.append((self.name, code.co_name, from_line, to_line)) + + +class BranchRecorder(JumpRecorder): + + event_type = E.BRANCH + name = "branch" + +class BranchRightRecorder(JumpRecorder): + + event_type = E.BRANCH_RIGHT + name = "branch right" + +class BranchLeftRecorder(JumpRecorder): + + event_type = E.BRANCH_LEFT + name = "branch left" + +class JumpOffsetRecorder: + + event_type = E.JUMP + name = "jump" + + def __init__(self, events, offsets=False): + self.events = events + + def __call__(self, code, from_, to): + self.events.append((self.name, code.co_name, from_, to)) + +class BranchLeftOffsetRecorder(JumpOffsetRecorder): + + event_type = E.BRANCH_LEFT + name = "branch left" + +class BranchRightOffsetRecorder(JumpOffsetRecorder): + + event_type = E.BRANCH_RIGHT + name = "branch right" + + +JUMP_AND_BRANCH_RECORDERS = JumpRecorder, BranchRecorder +JUMP_BRANCH_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder +FLOW_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder, ExceptionRecorder, ReturnRecorder + +BRANCHES_RECORDERS = BranchLeftRecorder, BranchRightRecorder +BRANCH_OFFSET_RECORDERS = BranchLeftOffsetRecorder, BranchRightOffsetRecorder + +class TestBranchAndJumpEvents(CheckEvents): + maxDiff = None + + @unittest.expectedFailure # TODO: RUSTPYTHON; - bytecode layout differs from CPython + def test_loop(self): + + def func(): + x = 1 + for a in range(2): + if a: + x = 4 + else: + x = 6 + 7 + + def whilefunc(n=0): + while n < 3: + n += 1 # line 2 + 3 + + self.check_events(func, recorders = JUMP_AND_BRANCH_RECORDERS, expected = [ + ('branch', 'func', 2, 2), + ('branch', 'func', 3, 6), + ('jump', 'func', 6, 2), + ('branch', 'func', 2, 2), + ('branch', 'func', 3, 4), + ('jump', 'func', 4, 2), + ('branch', 'func', 2, 7)]) + + self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func', 1), + ('line', 'func', 2), + ('branch', 'func', 2, 2), + ('line', 'func', 3), + ('branch', 'func', 3, 6), + ('line', 'func', 6), + ('jump', 'func', 6, 2), + ('line', 'func', 2), + ('branch', 'func', 2, 2), + ('line', 'func', 3), + ('branch', 'func', 3, 4), + ('line', 'func', 4), + ('jump', 'func', 4, 2), + ('line', 'func', 2), + ('branch', 'func', 2, 7), + ('line', 'func', 7), + ('line', 'get_events', 11)]) + + self.check_events(func, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'func', 2, 2), + ('branch right', 'func', 3, 6), + ('branch left', 'func', 2, 2), + ('branch left', 'func', 3, 4), + ('branch right', 'func', 2, 7)]) + + self.check_events(whilefunc, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'whilefunc', 1, 2), + ('branch left', 'whilefunc', 1, 2), + ('branch left', 'whilefunc', 1, 2), + ('branch right', 'whilefunc', 1, 3)]) + + self.check_events(func, recorders = BRANCH_OFFSET_RECORDERS, expected = [ + ('branch left', 'func', 28, 32), + ('branch right', 'func', 44, 58), + ('branch left', 'func', 28, 32), + ('branch left', 'func', 44, 50), + ('branch right', 'func', 28, 70)]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - bytecode layout differs from CPython + def test_except_star(self): + + class Foo: + def meth(self): + pass + + def func(): + try: + try: + raise KeyError + except* Exception as e: + f = Foo(); f.meth() + except KeyError: + pass + + + self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func', 1), + ('line', 'func', 2), + ('line', 'func', 3), + ('line', 'func', 4), + ('branch', 'func', 4, 4), + ('line', 'func', 5), + ('line', 'meth', 1), + ('jump', 'func', 5, '[offset=120]'), + ('branch', 'func', '[offset=124]', '[offset=130]'), + ('line', 'get_events', 11)]) + + self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func', 1), + ('line', 'func', 2), + ('line', 'func', 3), + ('raise', KeyError), + ('line', 'func', 4), + ('branch', 'func', 4, 4), + ('line', 'func', 5), + ('line', 'meth', 1), + ('return', 'meth', None), + ('jump', 'func', 5, '[offset=120]'), + ('branch', 'func', '[offset=124]', '[offset=130]'), + ('return', 'func', None), + ('line', 'get_events', 11)]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - bytecode layout differs from CPython + def test_while_offset_consistency(self): + + def foo(n=0): + while n<4: + pass + n += 1 + return None + + in_loop = ('branch left', 'foo', 10, 16) + exit_loop = ('branch right', 'foo', 10, 40) + self.check_events(foo, recorders = BRANCH_OFFSET_RECORDERS, expected = [ + in_loop, + in_loop, + in_loop, + in_loop, + exit_loop]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - bytecode layout differs from CPython + def test_async_for(self): + + def func(): + async def gen(): + yield 2 + yield 3 + + async def foo(): + async for y in gen(): + 2 + pass # line 3 + + try: + foo().send(None) + except StopIteration: + pass + + self.check_events(func, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'foo', 1, 1), + ('branch left', 'foo', 1, 1), + ('branch right', 'foo', 1, 3), + ('branch left', 'func', 12, 12)]) + + + @unittest.expectedFailure # TODO: RUSTPYTHON; - bytecode layout differs from CPython + def test_match(self): + + def func(v=1): + x = 0 + for v in range(4): + match v: + case 1: + x += 1 + case 2: + x += 2 + case _: + x += 3 + return x + + self.check_events(func, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'func', 2, 2), + ('branch right', 'func', 4, 6), + ('branch right', 'func', 6, 8), + ('branch left', 'func', 2, 2), + ('branch left', 'func', 4, 5), + ('branch left', 'func', 2, 2), + ('branch right', 'func', 4, 6), + ('branch left', 'func', 6, 7), + ('branch left', 'func', 2, 2), + ('branch right', 'func', 4, 6), + ('branch right', 'func', 6, 8), + ('branch right', 'func', 2, 10)]) + + def test_callback_set_frame_lineno(self): + def func(s: str) -> int: + if s.startswith("t"): + return 1 + else: + return 0 + + def callback(code, from_, to): + # try set frame.f_lineno + frame = inspect.currentframe() + while frame and frame.f_code is not code: + frame = frame.f_back + + self.assertIsNotNone(frame) + frame.f_lineno = frame.f_lineno + 1 # run next instruction + + sys.monitoring.set_local_events(TEST_TOOL, func.__code__, E.BRANCH_LEFT) + sys.monitoring.register_callback(TEST_TOOL, E.BRANCH_LEFT, callback) + + self.assertEqual(func("true"), 1) + + +class TestBranchConsistency(MonitoringTestBase, unittest.TestCase): + + def check_branches(self, run_func, test_func=None, tool=TEST_TOOL, recorders=BRANCH_OFFSET_RECORDERS): + if test_func is None: + test_func = run_func + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_local_events(tool, test_func.__code__, all_events) + run_func() + sys.monitoring.set_local_events(tool, test_func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + lefts = set() + rights = set() + for (src, left, right) in test_func.__code__.co_branches(): + lefts.add((src, left)) + rights.add((src, right)) + for event in event_list: + way, _, src, dest = event + if "left" in way: + self.assertIn((src, dest), lefts) + else: + self.assertIn("right", way) + self.assertIn((src, dest), rights) + finally: + sys.monitoring.set_local_events(tool, test_func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + def test_simple(self): + + def func(): + x = 1 + for a in range(2): + if a: + x = 4 + else: + x = 6 + 7 + + self.check_branches(func) + + def whilefunc(n=0): + while n < 3: + n += 1 # line 2 + 3 + + self.check_branches(whilefunc) + + def test_except_star(self): + + class Foo: + def meth(self): + pass + + def func(): + try: + try: + raise KeyError + except* Exception as e: + f = Foo(); f.meth() + except KeyError: + pass + + + self.check_branches(func) + + def test4(self): + + def foo(n=0): + while n<4: + pass + n += 1 + return None + + self.check_branches(foo) + + def test_async_for(self): + + async def gen(): + yield 2 + yield 3 + + async def foo(): + async for y in gen(): + 2 + pass # line 3 + + def func(): + try: + foo().send(None) + except StopIteration: + pass + + self.check_branches(func, foo) + + +class TestLoadSuperAttr(CheckEvents): + RECORDERS = CallRecorder, LineRecorder, CRaiseRecorder, CReturnRecorder + + def _exec(self, co): + d = {} + exec(co, d, d) + return d + + def _exec_super(self, codestr, optimized=False): + # The compiler checks for statically visible shadowing of the name + # `super`, and declines to emit `LOAD_SUPER_ATTR` if shadowing is found. + # So inserting `super = super` prevents the compiler from emitting + # `LOAD_SUPER_ATTR`, and allows us to test that monitoring events for + # `LOAD_SUPER_ATTR` are equivalent to those we'd get from the + # un-optimized `LOAD_GLOBAL super; CALL; LOAD_ATTR` form. + assignment = "x = 1" if optimized else "super = super" + codestr = f"{assignment}\n{textwrap.dedent(codestr)}" + co = compile(codestr, "", "exec") + # validate that we really do have a LOAD_SUPER_ATTR, only when optimized + self.assertEqual(self._has_load_super_attr(co), optimized) + return self._exec(co) + + def _has_load_super_attr(self, co): + has = any(instr.opname == "LOAD_SUPER_ATTR" for instr in dis.get_instructions(co)) + if not has: + has = any( + isinstance(c, types.CodeType) and self._has_load_super_attr(c) + for c in co.co_consts + ) + return has + + def _super_method_call(self, optimized=False): + codestr = """ + class A: + def method(self, x): + return x + + class B(A): + def method(self, x): + return super( + ).method( + x + ) + + b = B() + def f(): + return b.method(1) + """ + d = self._exec_super(codestr, optimized) + expected = [ + ('line', 'get_events', 10), + ('call', 'f', sys.monitoring.MISSING), + ('line', 'f', 1), + ('call', 'method', d["b"]), + ('line', 'method', 1), + ('call', 'super', sys.monitoring.MISSING), + ('C return', 'super', sys.monitoring.MISSING), + ('line', 'method', 2), + ('line', 'method', 3), + ('line', 'method', 2), + ('call', 'method', d["b"]), + ('line', 'method', 1), + ('line', 'method', 1), + ('line', 'get_events', 11), + ('call', 'set_events', 2), + ] + return d["f"], expected + + def test_method_call(self): + nonopt_func, nonopt_expected = self._super_method_call(optimized=False) + opt_func, opt_expected = self._super_method_call(optimized=True) + + self.check_events(nonopt_func, recorders=self.RECORDERS, expected=nonopt_expected) + self.check_events(opt_func, recorders=self.RECORDERS, expected=opt_expected) + + def _super_method_call_error(self, optimized=False): + codestr = """ + class A: + def method(self, x): + return x + + class B(A): + def method(self, x): + return super( + x, + self, + ).method( + x + ) + + b = B() + def f(): + try: + return b.method(1) + except TypeError: + pass + else: + assert False, "should have raised TypeError" + """ + d = self._exec_super(codestr, optimized) + expected = [ + ('line', 'get_events', 10), + ('call', 'f', sys.monitoring.MISSING), + ('line', 'f', 1), + ('line', 'f', 2), + ('call', 'method', d["b"]), + ('line', 'method', 1), + ('line', 'method', 2), + ('line', 'method', 3), + ('line', 'method', 1), + ('call', 'super', 1), + ('C raise', 'super', 1), + ('line', 'f', 3), + ('line', 'f', 4), + ('line', 'get_events', 11), + ('call', 'set_events', 2), + ] + return d["f"], expected + + def test_method_call_error(self): + nonopt_func, nonopt_expected = self._super_method_call_error(optimized=False) + opt_func, opt_expected = self._super_method_call_error(optimized=True) + + self.check_events(nonopt_func, recorders=self.RECORDERS, expected=nonopt_expected) + self.check_events(opt_func, recorders=self.RECORDERS, expected=opt_expected) + + def _super_attr(self, optimized=False): + codestr = """ + class A: + x = 1 + + class B(A): + def method(self): + return super( + ).x + + b = B() + def f(): + return b.method() + """ + d = self._exec_super(codestr, optimized) + expected = [ + ('line', 'get_events', 10), + ('call', 'f', sys.monitoring.MISSING), + ('line', 'f', 1), + ('call', 'method', d["b"]), + ('line', 'method', 1), + ('call', 'super', sys.monitoring.MISSING), + ('C return', 'super', sys.monitoring.MISSING), + ('line', 'method', 2), + ('line', 'method', 1), + ('line', 'get_events', 11), + ('call', 'set_events', 2) + ] + return d["f"], expected + + def test_attr(self): + nonopt_func, nonopt_expected = self._super_attr(optimized=False) + opt_func, opt_expected = self._super_attr(optimized=True) + + self.check_events(nonopt_func, recorders=self.RECORDERS, expected=nonopt_expected) + self.check_events(opt_func, recorders=self.RECORDERS, expected=opt_expected) + + def test_vs_other_type_call(self): + code_template = textwrap.dedent(""" + class C: + def method(self): + return {cls}().__repr__{call} + c = C() + def f(): + return c.method() + """) + + def get_expected(name, call_method, ns): + repr_arg = 0 if name == "int" else sys.monitoring.MISSING + return [ + ('line', 'get_events', 10), + ('call', 'f', sys.monitoring.MISSING), + ('line', 'f', 1), + ('call', 'method', ns["c"]), + ('line', 'method', 1), + ('call', name, sys.monitoring.MISSING), + ('C return', name, sys.monitoring.MISSING), + *( + [ + ('call', '__repr__', repr_arg), + ('C return', '__repr__', repr_arg), + ] if call_method else [] + ), + ('line', 'get_events', 11), + ('call', 'set_events', 2), + ] + + for call_method in [True, False]: + with self.subTest(call_method=call_method): + call_str = "()" if call_method else "" + code_super = code_template.format(cls="super", call=call_str) + code_int = code_template.format(cls="int", call=call_str) + co_super = compile(code_super, '', 'exec') + self.assertTrue(self._has_load_super_attr(co_super)) + ns_super = self._exec(co_super) + ns_int = self._exec(code_int) + + self.check_events( + ns_super["f"], + recorders=self.RECORDERS, + expected=get_expected("super", call_method, ns_super) + ) + self.check_events( + ns_int["f"], + recorders=self.RECORDERS, + expected=get_expected("int", call_method, ns_int) + ) + + +class TestSetGetEvents(MonitoringTestBase, unittest.TestCase): + + def test_global(self): + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 0) + sys.monitoring.set_events(TEST_TOOL2,0) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), 0) + + def test_local(self): + code = f1.__code__ + sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + sys.monitoring.set_local_events(TEST_TOOL, code, E.BRANCH) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.BRANCH_LEFT | E.BRANCH_RIGHT) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + sys.monitoring.set_local_events(TEST_TOOL2, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), E.PY_START) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) + sys.monitoring.set_local_events(TEST_TOOL2, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), 0) + +class TestUninitialized(unittest.TestCase, MonitoringTestBase): + + @staticmethod + def f(): + pass + + def test_get_local_events_uninitialized(self): + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, self.f.__code__), 0) + +class TestRegressions(MonitoringTestBase, unittest.TestCase): + + def test_105162(self): + caught = None + + def inner(): + nonlocal caught + try: + yield + except Exception: + caught = "inner" + yield + + def outer(): + nonlocal caught + try: + yield from inner() + except Exception: + caught = "outer" + yield + + def run(): + gen = outer() + gen.send(None) + gen.throw(Exception) + run() + self.assertEqual(caught, "inner") + caught = None + try: + sys.monitoring.set_events(TEST_TOOL, E.PY_RESUME) + run() + self.assertEqual(caught, "inner") + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_108390(self): + + class Foo: + def __init__(self, set_event): + if set_event: + sys.monitoring.set_events(TEST_TOOL, E.PY_RESUME) + + def make_foo_optimized_then_set_event(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD + 1): + Foo(i == _testinternalcapi.SPECIALIZATION_THRESHOLD) + + try: + make_foo_optimized_then_set_event() + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_gh108976(self): + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.LINE, lambda *args: sys.monitoring.set_events(0, 0)) + sys.monitoring.register_callback(0, E.INSTRUCTION, lambda *args: 0) + sys.monitoring.set_events(0, E.LINE | E.INSTRUCTION) + sys.monitoring.set_events(0, 0) + + def test_call_function_ex(self): + def f(a=1, b=2): + return a + b + args = (1, 2) + empty_args = [] + + call_data = [] + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.CALL, lambda code, offset, callable, arg0: call_data.append((callable, arg0))) + sys.monitoring.set_events(0, E.CALL) + f(*args) + f(*empty_args) + sys.monitoring.set_events(0, 0) + self.assertEqual(call_data[0], (f, 1)) + self.assertEqual(call_data[1], (f, sys.monitoring.MISSING)) + + def test_instruction_explicit_callback(self): + # gh-122247 + # Calling the instruction event callback explicitly should not + # crash CPython + def callback(code, instruction_offset): + pass + + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.register_callback(0, sys.monitoring.events.INSTRUCTION, callback) + sys.monitoring.set_events(0, sys.monitoring.events.INSTRUCTION) + callback(None, 0) # call the *same* handler while it is registered + sys.monitoring.restart_events() + sys.monitoring.set_events(0, 0) + + +class TestOptimizer(MonitoringTestBase, unittest.TestCase): + + def test_for_loop(self): + def test_func(x): + i = 0 + while i < x: + i += 1 + + code = test_func.__code__ + sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) + test_func(1000) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) + +class TestTier2Optimizer(CheckEvents): + + def test_monitoring_already_opimized_loop(self): + def test_func(recorder): + set_events = sys.monitoring.set_events + line = E.LINE + i = 0 + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD + 51): + # Turn on events without branching once i reaches _testinternalcapi.SPECIALIZATION_THRESHOLD. + set_events(TEST_TOOL, line * int(i >= _testinternalcapi.SPECIALIZATION_THRESHOLD)) + pass + pass + pass + + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = LineRecorder(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + try: + test_func(recorder) + finally: + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertGreater(len(events), 250) + +class TestMonitoringAtShutdown(unittest.TestCase): + + @unittest.expectedFailure # TODO: RUSTPYTHON; - requires subprocess support + def test_monitoring_live_at_shutdown(self): + # gh-115832: An object destructor running during the final GC of + # interpreter shutdown triggered an infinite loop in the + # instrumentation code. + script = test.support.findfile("_test_monitoring_shutdown.py") + script_helper.run_test_script(script) + + +class TestCApiEventGeneration(MonitoringTestBase, unittest.TestCase): + + class Scope: + def __init__(self, *args): + self.args = args + + def __enter__(self): + _testcapi.monitoring_enter_scope(*self.args) + + def __exit__(self, *args): + _testcapi.monitoring_exit_scope() + + def setUp(self): + super(TestCApiEventGeneration, self).setUp() + + capi = _testcapi + + self.codelike = capi.CodeLike(2) + + self.cases = [ + # (Event, function, *args) + ( 1, E.PY_START, capi.fire_event_py_start), + ( 1, E.PY_RESUME, capi.fire_event_py_resume), + ( 1, E.PY_YIELD, capi.fire_event_py_yield, 10), + ( 1, E.PY_RETURN, capi.fire_event_py_return, 20), + ( 2, E.CALL, capi.fire_event_call, callable, 40), + ( 1, E.JUMP, capi.fire_event_jump, 60), + ( 1, E.BRANCH_RIGHT, capi.fire_event_branch_right, 70), + ( 1, E.BRANCH_LEFT, capi.fire_event_branch_left, 80), + ( 1, E.PY_THROW, capi.fire_event_py_throw, ValueError(1)), + ( 1, E.RAISE, capi.fire_event_raise, ValueError(2)), + ( 1, E.EXCEPTION_HANDLED, capi.fire_event_exception_handled, ValueError(5)), + ( 1, E.PY_UNWIND, capi.fire_event_py_unwind, ValueError(6)), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, 7), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, StopIteration(8)), + ] + + self.EXPECT_RAISED_EXCEPTION = [E.PY_THROW, E.RAISE, E.EXCEPTION_HANDLED, E.PY_UNWIND] + + + def check_event_count(self, event, func, args, expected, callback_raises=None): + class Counter: + def __init__(self, callback_raises): + self.callback_raises = callback_raises + self.count = 0 + + def __call__(self, *args): + self.count += 1 + if self.callback_raises: + exc = self.callback_raises + self.callback_raises = None + raise exc + + try: + counter = Counter(callback_raises) + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + try: + func(*args) + except ValueError as e: + self.assertIsInstance(expected, ValueError) + self.assertEqual(str(e), str(expected)) + return + else: + self.assertEqual(counter.count, expected) + + prev = sys.monitoring.register_callback(TEST_TOOL, event, None) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, 0) + self.assertEqual(prev, counter) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_fire_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args) + self.check_event_count(event, function, args_, expected) + + def test_missing_exception(self): + for _, event, function, *args in self.cases: + if event not in self.EXPECT_RAISED_EXCEPTION: + continue + assert args and isinstance(args[-1], BaseException) + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args[:-1]) + (None,) + evt = int(math.log2(event)) + expected = ValueError(f"Firing event {evt} with no exception set") + self.check_event_count(event, function, args_, expected) + + def test_fire_event_failing_callback(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args) + exc = OSError(42) + with self.assertRaises(type(exc)): + self.check_event_count(event, function, args_, expected, + callback_raises=exc) + + + CANNOT_DISABLE = { E.PY_THROW, E.RAISE, E.RERAISE, + E.EXCEPTION_HANDLED, E.PY_UNWIND } + + def check_disable(self, event, func, args, expected): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.disable = True + if event in self.CANNOT_DISABLE: + # use try-except rather then assertRaises to avoid + # events from framework code + try: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + except ValueError: + pass + else: + self.Error("Expected a ValueError") + else: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected - 1) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_disable_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(2) + with self.subTest(function.__name__): + args_ = (self.codelike, 0) + tuple(args) + self.check_disable(event, function, args_, expected) + + def test_enter_scope_two_events(self): + try: + yield_counter = CounterWithDisable() + unwind_counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_YIELD, yield_counter) + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, unwind_counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_YIELD | E.PY_UNWIND) + + yield_value = int(math.log2(E.PY_YIELD)) + unwind_value = int(math.log2(E.PY_UNWIND)) + cl = _testcapi.CodeLike(2) + common_args = (cl, 0) + with self.Scope(cl, yield_value, unwind_value): + yield_counter.count = 0 + unwind_counter.count = 0 + + _testcapi.fire_event_py_unwind(*common_args, ValueError(42)) + assert(yield_counter.count == 0) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 1) + assert(unwind_counter.count == 1) + + yield_counter.disable = True + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + finally: + sys.monitoring.set_events(TEST_TOOL, 0) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 205facd65bd..75e5deb4541 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -2865,6 +2865,9 @@ impl Compiler { // Normal path jumps here to skip exception path blocks let end_block = self.new_block(); + // Emit NOP at the try: line so LINE events fire for it + emit!(self, Instruction::Nop); + // Setup a finally block if we have a finally statement. // Push fblock with handler info for exception table generation // IMPORTANT: handler goes to finally_except_block (exception path), not finally_block @@ -3018,8 +3021,10 @@ impl Compiler { type_, name, body, + range: handler_range, .. }) = &handler; + self.set_source_range(*handler_range); let next_handler = self.new_block(); // If we gave a typ, @@ -3313,6 +3318,9 @@ impl Compiler { }; let exit_block = self.new_block(); + // Emit NOP at the try: line so LINE events fire for it + emit!(self, Instruction::Nop); + // Push fblock with handler info for exception table generation if !finalbody.is_empty() { emit!( @@ -3743,6 +3751,9 @@ impl Compiler { is_async: bool, funcflags: bytecode::MakeFunctionFlags, ) -> CompileResult<()> { + // Save source range so MAKE_FUNCTION gets the `def` line, not the body's last line + let saved_range = self.current_source_range; + // Always enter function scope self.enter_function(name, parameters)?; self.current_code_info() @@ -3796,6 +3807,9 @@ impl Compiler { let code = self.exit_scope(); self.ctx = prev_ctx; + // Restore source range so MAKE_FUNCTION is attributed to the `def` line + self.set_source_range(saved_range); + // Create function object with closure self.make_closure(code, funcflags)?; @@ -5169,15 +5183,22 @@ impl Compiler { if is_async { emit!(self, Instruction::EndAsyncFor); } else { - // END_FOR + POP_ITER pattern (CPython 3.14) - // FOR_ITER jumps to END_FOR, but VM skips it (+1) to reach POP_ITER + // END_FOR + POP_ITER are on the `for` line, not the body's last line + let saved_range = self.current_source_range; + self.set_source_range(iter.range()); emit!(self, Instruction::EndFor); emit!(self, Instruction::PopIter); + self.set_source_range(saved_range); } self.compile_statements(orelse)?; self.switch_to_block(after_block); + // Restore source range to the `for` line so any implicit return + // (LOAD_CONST None, RETURN_VALUE) is attributed to the `for` line, + // not the loop body's last line. + self.set_source_range(iter.range()); + self.leave_conditional_block(); Ok(()) } diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index e44c7223f2c..b2be3eca16c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -244,15 +244,33 @@ impl CodeInfo { let mut locations = Vec::new(); let mut linetable_locations: Vec = Vec::new(); - // Convert pseudo ops and remove resulting NOPs + // Convert pseudo ops and remove resulting NOPs (keep line-marker NOPs) convert_pseudo_ops(&mut blocks, varname_cache.len() as u32); for block in blocks .iter_mut() .filter(|b| b.next != BlockIdx::NULL || !b.instructions.is_empty()) { - block + // Collect lines that have non-NOP instructions in this block + let non_nop_lines: std::collections::HashSet<_> = block .instructions - .retain(|ins| !matches!(ins.instr.real(), Some(Instruction::Nop))); + .iter() + .filter(|ins| !matches!(ins.instr.real(), Some(Instruction::Nop))) + .map(|ins| ins.location.line) + .collect(); + let mut kept_nop_lines: std::collections::HashSet = + std::collections::HashSet::new(); + block.instructions.retain(|ins| { + if matches!(ins.instr.real(), Some(Instruction::Nop)) { + let line = ins.location.line; + // Remove if another instruction covers this line, + // or if we already kept a NOP for this line + if non_nop_lines.contains(&line) || kept_nop_lines.contains(&line) { + return false; + } + kept_nop_lines.insert(line); + } + true + }); } let mut block_to_offset = vec![Label(0); blocks.len()]; @@ -495,9 +513,14 @@ impl CodeInfo { let tuple_const = ConstantData::Tuple { elements }; let (const_idx, _) = self.metadata.consts.insert_full(tuple_const); - // Replace preceding LOAD instructions with NOP + // Replace preceding LOAD instructions with NOP, using the + // BUILD_TUPLE location so remove_nops() treats them as + // same-line and removes them (multi-line tuple literals + // would otherwise leave line-introducing NOPs behind). + let folded_loc = block.instructions[i].location; for j in start_idx..i { block.instructions[j].instr = Instruction::Nop.into(); + block.instructions[j].location = folded_loc; } // Replace BUILD_TUPLE with LOAD_CONST @@ -667,12 +690,23 @@ impl CodeInfo { } } - /// Remove NOP instructions from all blocks + /// Remove NOP instructions from all blocks, but keep NOPs that introduce + /// a new source line (they serve as line markers for monitoring LINE events). fn remove_nops(&mut self) { for block in &mut self.blocks { - block - .instructions - .retain(|ins| !matches!(ins.instr.real(), Some(Instruction::Nop))); + let mut prev_line = None; + block.instructions.retain(|ins| { + if matches!(ins.instr.real(), Some(Instruction::Nop)) { + let line = ins.location.line; + if prev_line == Some(line) { + // Same line as previous instruction — safe to remove + return false; + } + // This NOP introduces a new line — keep it + } + prev_line = Some(ins.location.line); + true + }); } } diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap index c23ef890daf..8562e57d208 100644 --- a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap @@ -1,11 +1,9 @@ --- source: crates/codegen/src/compile.rs -assertion_line: 9046 expression: "compile_exec(\"\\\nasync def test():\n for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n with self.subTest(type=type(stop_exc)):\n try:\n async with egg():\n raise stop_exc\n except Exception as ex:\n self.assertIs(ex, stop_exc)\n else:\n self.fail(f'{stop_exc} was suppressed')\n\")" --- 1 0 RESUME (0) - - 3 1 LOAD_CONST (): 1 0 RETURN_GENERATOR + 1 LOAD_CONST (): 1 0 RETURN_GENERATOR 1 POP_TOP 2 RESUME (0) @@ -19,7 +17,7 @@ expression: "compile_exec(\"\\\nasync def test():\n for stop_exc in (StopIter 10 CALL (1) 11 BUILD_TUPLE (2) 12 GET_ITER - >> 13 FOR_ITER (141) + >> 13 FOR_ITER (144) 14 STORE_FAST (0, stop_exc) 3 15 LOAD_GLOBAL (2, self) @@ -38,127 +36,132 @@ expression: "compile_exec(\"\\\nasync def test():\n for stop_exc in (StopIter 28 CALL (0) 29 POP_TOP - 5 30 LOAD_GLOBAL (5, egg) - 31 PUSH_NULL - 32 CALL (0) - 33 COPY (1) - 34 LOAD_SPECIAL (__aexit__) - 35 SWAP (2) - 36 LOAD_SPECIAL (__aenter__) - 37 PUSH_NULL - 38 CALL (0) - 39 GET_AWAITABLE (1) - 40 LOAD_CONST (None) - >> 41 SEND (46) - 42 YIELD_VALUE (1) - 43 RESUME (3) - 44 JUMP_BACKWARD_NO_INTERRUPT(41) - 45 CLEANUP_THROW - >> 46 END_SEND - 47 POP_TOP + 4 30 NOP + + 5 31 LOAD_GLOBAL (5, egg) + 32 PUSH_NULL + 33 CALL (0) + 34 COPY (1) + 35 LOAD_SPECIAL (__aexit__) + 36 SWAP (2) + 37 LOAD_SPECIAL (__aenter__) + 38 PUSH_NULL + 39 CALL (0) + 40 GET_AWAITABLE (1) + 41 LOAD_CONST (None) + >> 42 SEND (47) + 43 YIELD_VALUE (1) + 44 RESUME (3) + 45 JUMP_BACKWARD_NO_INTERRUPT(42) + 46 CLEANUP_THROW + >> 47 END_SEND + 48 POP_TOP - 6 48 LOAD_FAST (0, stop_exc) - 49 RAISE_VARARGS (Raise) + 6 49 LOAD_FAST (0, stop_exc) + 50 RAISE_VARARGS (Raise) + 51 NOP - 5 50 PUSH_NULL - 51 LOAD_CONST (None) - 52 LOAD_CONST (None) + 5 52 PUSH_NULL 53 LOAD_CONST (None) - 54 CALL (3) - 55 GET_AWAITABLE (2) - 56 LOAD_CONST (None) - >> 57 SEND (62) - 58 YIELD_VALUE (1) - 59 RESUME (3) - 60 JUMP_BACKWARD_NO_INTERRUPT(57) - 61 CLEANUP_THROW - >> 62 END_SEND - 63 POP_TOP - 64 JUMP_FORWARD (86) - 65 PUSH_EXC_INFO - 66 WITH_EXCEPT_START - 67 GET_AWAITABLE (2) - 68 LOAD_CONST (None) - >> 69 SEND (74) - 70 YIELD_VALUE (1) - 71 RESUME (3) - 72 JUMP_BACKWARD_NO_INTERRUPT(69) - 73 CLEANUP_THROW - >> 74 END_SEND - 75 TO_BOOL - 76 POP_JUMP_IF_TRUE (78) - 77 RERAISE (2) - >> 78 POP_TOP - 79 POP_EXCEPT - 80 POP_TOP - 81 POP_TOP - 82 JUMP_FORWARD (86) - 83 COPY (3) - 84 POP_EXCEPT - 85 RERAISE (1) - >> 86 JUMP_FORWARD (112) - 87 PUSH_EXC_INFO + 54 LOAD_CONST (None) + 55 LOAD_CONST (None) + 56 CALL (3) + 57 GET_AWAITABLE (2) + 58 LOAD_CONST (None) + >> 59 SEND (64) + 60 YIELD_VALUE (1) + 61 RESUME (3) + 62 JUMP_BACKWARD_NO_INTERRUPT(59) + 63 CLEANUP_THROW + >> 64 END_SEND + 65 POP_TOP + 66 JUMP_FORWARD (88) + 67 PUSH_EXC_INFO + 68 WITH_EXCEPT_START + 69 GET_AWAITABLE (2) + 70 LOAD_CONST (None) + >> 71 SEND (76) + 72 YIELD_VALUE (1) + 73 RESUME (3) + 74 JUMP_BACKWARD_NO_INTERRUPT(71) + 75 CLEANUP_THROW + >> 76 END_SEND + 77 TO_BOOL + 78 POP_JUMP_IF_TRUE (80) + 79 RERAISE (2) + >> 80 POP_TOP + 81 POP_EXCEPT + 82 POP_TOP + 83 POP_TOP + 84 JUMP_FORWARD (88) + 85 COPY (3) + 86 POP_EXCEPT + 87 RERAISE (1) + >> 88 JUMP_FORWARD (114) + 89 PUSH_EXC_INFO + + 7 90 LOAD_GLOBAL (6, Exception) + 91 CHECK_EXC_MATCH + 92 POP_JUMP_IF_FALSE (110) + 93 STORE_FAST (1, ex) - 7 88 LOAD_GLOBAL (6, Exception) - 89 CHECK_EXC_MATCH - 90 POP_JUMP_IF_FALSE (108) - 91 STORE_FAST (1, ex) + 8 94 LOAD_GLOBAL (2, self) + 95 LOAD_ATTR (15, assertIs, method=true) + 96 LOAD_FAST (1, ex) + 97 LOAD_FAST (0, stop_exc) + 98 CALL (2) + 99 POP_TOP + 100 JUMP_FORWARD (105) + 101 LOAD_CONST (None) + 102 STORE_FAST (1, ex) + 103 DELETE_FAST (1, ex) + 104 RAISE_VARARGS (ReraiseFromStack) + >> 105 POP_EXCEPT + 106 LOAD_CONST (None) + 107 STORE_FAST (1, ex) + 108 DELETE_FAST (1, ex) + 109 JUMP_FORWARD (122) + >> 110 RAISE_VARARGS (ReraiseFromStack) + 111 COPY (3) + 112 POP_EXCEPT + 113 RAISE_VARARGS (ReraiseFromStack) - 8 92 LOAD_GLOBAL (2, self) - 93 LOAD_ATTR (15, assertIs, method=true) - 94 LOAD_FAST (1, ex) - 95 LOAD_FAST (0, stop_exc) - 96 CALL (2) - 97 POP_TOP - 98 JUMP_FORWARD (103) - 99 LOAD_CONST (None) - 100 STORE_FAST (1, ex) - 101 DELETE_FAST (1, ex) - 102 RAISE_VARARGS (ReraiseFromStack) - >> 103 POP_EXCEPT - 104 LOAD_CONST (None) - 105 STORE_FAST (1, ex) - 106 DELETE_FAST (1, ex) - 107 JUMP_FORWARD (120) - >> 108 RAISE_VARARGS (ReraiseFromStack) - 109 COPY (3) - 110 POP_EXCEPT - 111 RAISE_VARARGS (ReraiseFromStack) + 10 >> 114 LOAD_GLOBAL (2, self) + 115 LOAD_ATTR (17, fail, method=true) + 116 LOAD_FAST_BORROW (0, stop_exc) + 117 FORMAT_SIMPLE + 118 LOAD_CONST (" was suppressed") + 119 BUILD_STRING (2) + 120 CALL (1) + 121 POP_TOP + >> 122 NOP - 10 >> 112 LOAD_GLOBAL (2, self) - 113 LOAD_ATTR (17, fail, method=true) - 114 LOAD_FAST_BORROW (0, stop_exc) - 115 FORMAT_SIMPLE - 116 LOAD_CONST (" was suppressed") - 117 BUILD_STRING (2) - 118 CALL (1) - 119 POP_TOP + 3 123 PUSH_NULL + 124 LOAD_CONST (None) + 125 LOAD_CONST (None) + 126 LOAD_CONST (None) + 127 CALL (3) + 128 POP_TOP + 129 JUMP_FORWARD (143) + 130 PUSH_EXC_INFO + 131 WITH_EXCEPT_START + 132 TO_BOOL + 133 POP_JUMP_IF_TRUE (135) + 134 RERAISE (2) + >> 135 POP_TOP + 136 POP_EXCEPT + 137 POP_TOP + 138 POP_TOP + 139 JUMP_FORWARD (143) + 140 COPY (3) + 141 POP_EXCEPT + 142 RERAISE (1) + >> 143 JUMP_BACKWARD (13) - 3 >> 120 PUSH_NULL - 121 LOAD_CONST (None) - 122 LOAD_CONST (None) - 123 LOAD_CONST (None) - 124 CALL (3) - 125 POP_TOP - 126 JUMP_FORWARD (140) - 127 PUSH_EXC_INFO - 128 WITH_EXCEPT_START - 129 TO_BOOL - 130 POP_JUMP_IF_TRUE (132) - 131 RERAISE (2) - >> 132 POP_TOP - 133 POP_EXCEPT - 134 POP_TOP - 135 POP_TOP - 136 JUMP_FORWARD (140) - 137 COPY (3) - 138 POP_EXCEPT - 139 RERAISE (1) - >> 140 JUMP_BACKWARD (13) - >> 141 END_FOR - 142 POP_ITER - 143 LOAD_CONST (None) - 144 RETURN_VALUE + 2 >> 144 END_FOR + 145 POP_ITER + 146 LOAD_CONST (None) + 147 RETURN_VALUE 2 MAKE_FUNCTION 3 STORE_NAME (0, test) diff --git a/crates/derive-impl/src/pymodule.rs b/crates/derive-impl/src/pymodule.rs index ed86d142cef..293dadbd110 100644 --- a/crates/derive-impl/src/pymodule.rs +++ b/crates/derive-impl/src/pymodule.rs @@ -194,6 +194,60 @@ pub fn impl_pymodule(args: PyModuleArgs, module_item: Item) -> Result = Vec::new(); + for item in items.iter() { + if let Item::Mod(item_mod) = item { + let r = (|| -> Result<()> { + let attr = match item_mod + .attrs + .iter() + .find(|a| a.path().is_ident("pymodule")) + { + Some(attr) => attr, + None => return Ok(()), + }; + let args_tokens = match &attr.meta { + syn::Meta::Path(_) => TokenStream::new(), + syn::Meta::List(list) => list.tokens.clone(), + _ => return Ok(()), + }; + let mod_args: PyModuleArgs = syn::parse2(args_tokens)?; + let fake_ident = Ident::new("pymodule", attr.span()); + let mod_meta = ModuleItemMeta::from_nested( + item_mod.ident.clone(), + fake_ident, + mod_args.metas.into_iter(), + )?; + if mod_meta.sub()? { + return Ok(()); + } + let py_name = mod_meta.simple_name()?; + let mod_ident = &item_mod.ident; + let cfgs: Vec<_> = item_mod + .attrs + .iter() + .filter(|a| a.path().is_ident("cfg")) + .cloned() + .collect(); + submodule_inits.push(quote! { + #(#cfgs)* + { + let child_def = #mod_ident::module_def(ctx); + let child = child_def.create_module(vm).unwrap(); + ::rustpython_vm::builtins::PyModule::__init_dict_from_def(vm, &child); + child.__init_methods(vm).unwrap(); + #mod_ident::module_exec(vm, &child).unwrap(); + let child: ::rustpython_vm::PyObjectRef = child.into(); + vm.__module_set_attr(module, ctx.intern_str(#py_name), child).unwrap(); + } + }); + Ok(()) + })(); + context.errors.ok_or_push(r); + } + } + // append additional items let module_name = context.name.as_str(); let function_items = context.function_items.validate()?; @@ -316,6 +370,7 @@ pub fn impl_pymodule(args: PyModuleArgs, module_item: Item) -> Result { lasti: &self.lasti, object: self, state: &mut state, + monitoring_mask: 0, }; f(exec) } @@ -379,6 +380,7 @@ impl Py { lasti: &self.lasti, object: self, state: &mut state, + monitoring_mask: 0, }; exec.yield_from_target().map(PyObject::to_owned) } @@ -414,6 +416,8 @@ struct ExecutingFrame<'a> { object: &'a Py, lasti: &'a Lasti, state: &'a mut FrameState, + /// Cached monitoring events mask. Reloaded at Resume instruction only, + monitoring_mask: u32, } impl fmt::Debug for ExecutingFrame<'_> { @@ -464,6 +468,7 @@ impl ExecutingFrame<'_> { let instructions = &self.code.instructions; let mut arg_state = bytecode::OpArgState::default(); let mut prev_line: u32 = 0; + self.monitoring_mask = vm.state.monitoring_events.load(); loop { let idx = self.lasti() as usize; // Fire 'line' trace event when line number changes. @@ -481,6 +486,25 @@ impl ExecutingFrame<'_> { let bytecode::CodeUnit { op, arg } = instructions[idx]; let arg = arg_state.extend(arg); let mut do_extend_arg = false; + + // Always track current line for LINE monitoring, even when + // monitoring is off. This prevents spurious LINE events when + // monitoring is enabled mid-function. + if !matches!(op, Instruction::Resume { .. } | Instruction::ExtendedArg) { + let line = self.code.locations[idx].0.line.get() as u32; + if self.monitoring_mask != 0 { + use crate::stdlib::sys::monitoring; + let offset = idx as u32 * 2; + if self.monitoring_mask & monitoring::EVENT_LINE != 0 && line != prev_line { + monitoring::fire_line(vm, self.code, offset, line)?; + } + if self.monitoring_mask & monitoring::EVENT_INSTRUCTION != 0 { + monitoring::fire_instruction(vm, self.code, offset)?; + } + } + prev_line = line; + } + let result = self.execute_instruction(op, arg, &mut do_extend_arg, vm); match result { Ok(None) => {} @@ -567,10 +591,57 @@ impl ExecutingFrame<'_> { ) ); + // Fire RAISE or RERAISE monitoring event. + // fire_reraise internally deduplicates: only the first + // re-raise after each EXCEPTION_HANDLED fires the event. + // If the callback raises (e.g. ValueError for illegal DISABLE), + // replace the original exception. + let exception = { + use crate::stdlib::sys::monitoring; + if is_reraise { + if self.monitoring_mask & monitoring::EVENT_RERAISE != 0 { + let offset = idx as u32 * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + match monitoring::fire_reraise(vm, self.code, offset, &exc_obj) { + Ok(()) => exception, + Err(monitor_exc) => monitor_exc, + } + } else { + exception + } + } else if self.monitoring_mask & monitoring::EVENT_RAISE != 0 { + let offset = idx as u32 * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + match monitoring::fire_raise(vm, self.code, offset, &exc_obj) { + Ok(()) => exception, + Err(monitor_exc) => monitor_exc, + } + } else { + exception + } + }; + match handle_exception(self, exception, idx, is_reraise, is_new_raise, vm) { Ok(None) => {} Ok(Some(result)) => break Ok(result), Err(exception) => { + // Fire PY_UNWIND: exception escapes this frame + let exception = if self.monitoring_mask + & crate::stdlib::sys::monitoring::EVENT_PY_UNWIND + != 0 + { + let offset = idx as u32 * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + match crate::stdlib::sys::monitoring::fire_py_unwind( + vm, self.code, offset, &exc_obj, + ) { + Ok(()) => exception, + Err(monitor_exc) => monitor_exc, + } + } else { + exception + }; + // Restore lasti from traceback so frame.f_lineno matches tb_lineno // The traceback was created with the correct lasti when exception // was first raised, but frame.lasti may have changed during cleanup @@ -637,6 +708,7 @@ impl ExecutingFrame<'_> { exc_val: PyObjectRef, exc_tb: PyObjectRef, ) -> PyResult { + self.monitoring_mask = vm.state.monitoring_events.load(); if let Some(jen) = self.yield_from_target() { // Check if the exception is GeneratorExit (type or instance). // For GeneratorExit, close the sub-iterator instead of throwing. @@ -747,6 +819,33 @@ impl ExecutingFrame<'_> { exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); } + // Fire PY_THROW and RAISE events before raising the exception in the + // generator. do_monitor_exc in CPython replaces the active exception + // when a callback fails, so we mirror that here. + let exception = { + use crate::stdlib::sys::monitoring; + let exception = if self.monitoring_mask & monitoring::EVENT_PY_THROW != 0 { + let offset = idx as u32 * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + match monitoring::fire_py_throw(vm, self.code, offset, &exc_obj) { + Ok(()) => exception, + Err(monitor_exc) => monitor_exc, + } + } else { + exception + }; + if self.monitoring_mask & monitoring::EVENT_RAISE != 0 { + let offset = idx as u32 * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + match monitoring::fire_raise(vm, self.code, offset, &exc_obj) { + Ok(()) => exception, + Err(monitor_exc) => monitor_exc, + } + } else { + exception + } + }; + // when raising an exception, set __context__ to the current exception // This is done in _PyErr_SetObject vm.contextualize_exception(&exception); @@ -758,7 +857,26 @@ impl ExecutingFrame<'_> { match self.unwind_blocks(vm, UnwindReason::Raising { exception }) { Ok(None) => self.run(vm), Ok(Some(result)) => Ok(result), - Err(exception) => Err(exception), + Err(exception) => { + // Fire PY_UNWIND: exception escapes the generator frame. + // do_monitor_exc replaces the exception on callback failure. + let exception = if self.monitoring_mask + & crate::stdlib::sys::monitoring::EVENT_PY_UNWIND + != 0 + { + let offset = idx as u32 * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + match crate::stdlib::sys::monitoring::fire_py_unwind( + vm, self.code, offset, &exc_obj, + ) { + Ok(()) => exception, + Err(monitor_exc) => monitor_exc, + } + } else { + exception + }; + Err(exception) + } } } @@ -1336,7 +1454,18 @@ impl ExecutingFrame<'_> { Ok(None) } Instruction::JumpBackward { target } => { - self.jump(target.get(arg)); + let src_offset = (self.lasti() - 1) * 2; + let dest = target.get(arg); + self.jump(dest); + // JUMP events fire only for backward jumps (loop iterations) + if self.monitoring_mask & crate::stdlib::sys::monitoring::EVENT_JUMP != 0 { + crate::stdlib::sys::monitoring::fire_jump( + vm, + self.code, + src_offset, + dest.0 * 2, + )?; + } Ok(None) } Instruction::JumpBackwardNoInterrupt { target } => { @@ -1934,16 +2063,24 @@ impl ExecutingFrame<'_> { Instruction::PopJumpIfTrue { target } => self.pop_jump_if(vm, target.get(arg), true), Instruction::PopJumpIfNone { target } => { let value = self.pop_value(); - if vm.is_none(&value) { - self.jump(target.get(arg)); + let branch_taken = vm.is_none(&value); + let target = target.get(arg); + let src_offset = (self.lasti() - 1) * 2; + if branch_taken { + self.jump(target); } + self.fire_branch_event(vm, src_offset, branch_taken, target)?; Ok(None) } Instruction::PopJumpIfNotNone { target } => { let value = self.pop_value(); - if !vm.is_none(&value) { - self.jump(target.get(arg)); + let branch_taken = !vm.is_none(&value); + let target = target.get(arg); + let src_offset = (self.lasti() - 1) * 2; + if branch_taken { + self.jump(target); } + self.fire_branch_event(vm, src_offset, branch_taken, target)?; Ok(None) } Instruction::PopTop => { @@ -1968,19 +2105,25 @@ impl ExecutingFrame<'_> { } Instruction::RaiseVarargs { kind } => self.execute_raise(vm, kind.get(arg)), Instruction::Resume { arg: resume_arg } => { - // Resume execution after yield, await, or at function start - // In CPython, this checks instrumentation and eval breaker - // For now, we just check for signals/interrupts - let _resume_type = resume_arg.get(arg); - - // Check for interrupts if not resuming from yield_from - // if resume_type < bytecode::ResumeType::AfterYieldFrom as u32 { - // vm.check_signals()?; - // } + use crate::stdlib::sys::monitoring; + let resume_type = resume_arg.get(arg); + self.monitoring_mask = vm.state.monitoring_events.load(); + let offset = (self.lasti() - 1) * 2; + if resume_type == 0 { + if self.monitoring_mask & monitoring::EVENT_PY_START != 0 { + monitoring::fire_py_start(vm, self.code, offset)?; + } + } else if self.monitoring_mask & monitoring::EVENT_PY_RESUME != 0 { + monitoring::fire_py_resume(vm, self.code, offset)?; + } Ok(None) } Instruction::ReturnValue => { let value = self.pop_value(); + if self.monitoring_mask & crate::stdlib::sys::monitoring::EVENT_PY_RETURN != 0 { + let offset = (self.lasti() - 1) * 2; + crate::stdlib::sys::monitoring::fire_py_return(vm, self.code, offset, &value)?; + } self.unwind_blocks(vm, UnwindReason::Returning { value }) } Instruction::SetAdd { i } => { @@ -2019,8 +2162,10 @@ impl ExecutingFrame<'_> { vm.set_exception(Some(exc_ref.to_owned())); } + // Complete stack operations self.push_value(prev_exc); self.push_value(exc); + Ok(None) } Instruction::CheckExcMatch => { @@ -2201,6 +2346,10 @@ impl ExecutingFrame<'_> { } Instruction::YieldValue { arg: oparg } => { let value = self.pop_value(); + if self.monitoring_mask & crate::stdlib::sys::monitoring::EVENT_PY_YIELD != 0 { + let offset = (self.lasti() - 1) * 2; + crate::stdlib::sys::monitoring::fire_py_yield(vm, self.code, offset, &value)?; + } // arg=0: direct yield (wrapped for async generators) // arg=1: yield from await/yield-from (NOT wrapped) let wrap = oparg.get(arg) == 0; @@ -2529,6 +2678,23 @@ impl ExecutingFrame<'_> { if let Some(entry) = bytecode::find_exception_handler(&self.code.exceptiontable, offset) { + // Fire EXCEPTION_HANDLED before setting up handler. + // If the callback raises, the handler is NOT set up and the + // new exception propagates instead. + if self.monitoring_mask + & crate::stdlib::sys::monitoring::EVENT_EXCEPTION_HANDLED + != 0 + { + let byte_offset = offset * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + crate::stdlib::sys::monitoring::fire_exception_handled( + vm, + self.code, + byte_offset, + &exc_obj, + )?; + } + // 1. Pop stack to entry.depth while self.state.stack.len() > entry.depth as usize { self.state.stack.pop(); @@ -2729,6 +2895,8 @@ impl ExecutingFrame<'_> { #[inline] fn execute_call(&mut self, args: FuncArgs, vm: &VirtualMachine) -> FrameResult { + use crate::stdlib::sys::monitoring; + // Stack: [callable, self_or_null, ...] let self_or_null = self.pop_value_opt(); // Option let callable = self.pop_value(); @@ -2747,9 +2915,50 @@ impl ExecutingFrame<'_> { args }; - let value = callable.call(final_args, vm)?; - self.push_value(value); - Ok(None) + let is_python_call = callable.downcast_ref::().is_some(); + + // Compute arg0 once for CALL, C_RETURN, and C_RAISE events + let call_arg0 = if self.monitoring_mask & monitoring::EVENT_CALL != 0 { + let arg0 = final_args + .args + .first() + .cloned() + .unwrap_or_else(|| monitoring::get_missing(vm)); + let offset = (self.lasti() - 1) * 2; + monitoring::fire_call(vm, self.code, offset, &callable, arg0.clone())?; + Some(arg0) + } else { + None + }; + + match callable.call(final_args, vm) { + Ok(value) => { + if let Some(arg0) = call_arg0 + && !is_python_call + { + let offset = (self.lasti() - 1) * 2; + monitoring::fire_c_return(vm, self.code, offset, &callable, arg0)?; + } + self.push_value(value); + Ok(None) + } + Err(exc) => { + // call_instrumentation_vector_protected replaces the active + // exception when the callback itself raises. + let exc = if let Some(arg0) = call_arg0 + && !is_python_call + { + let offset = (self.lasti() - 1) * 2; + match monitoring::fire_c_raise(vm, self.code, offset, &callable, arg0) { + Ok(()) => exc, + Err(monitor_exc) => monitor_exc, + } + } else { + exc + }; + Err(exc) + } + } } fn execute_raise(&mut self, vm: &VirtualMachine, kind: bytecode::RaiseKind) -> FrameResult { @@ -2878,6 +3087,43 @@ impl ExecutingFrame<'_> { self.update_lasti(|i| *i = target_pc); } + #[inline] + fn fire_branch_event( + &self, + vm: &VirtualMachine, + src_offset: u32, + branch_taken: bool, + target: bytecode::Label, + ) -> PyResult<()> { + if self.monitoring_mask + & (crate::stdlib::sys::monitoring::EVENT_BRANCH_LEFT + | crate::stdlib::sys::monitoring::EVENT_BRANCH_RIGHT) + != 0 + { + let dest_offset = if branch_taken { + target.0 * 2 + } else { + self.lasti() * 2 + }; + if branch_taken { + crate::stdlib::sys::monitoring::fire_branch_right( + vm, + self.code, + src_offset, + dest_offset, + )?; + } else { + crate::stdlib::sys::monitoring::fire_branch_left( + vm, + self.code, + src_offset, + dest_offset, + )?; + } + } + Ok(()) + } + #[inline] fn pop_jump_if( &mut self, @@ -2887,9 +3133,12 @@ impl ExecutingFrame<'_> { ) -> FrameResult { let obj = self.pop_value(); let value = obj.try_to_bool(vm)?; - if value == flag { + let branch_taken = value == flag; + let src_offset = (self.lasti() - 1) * 2; + if branch_taken { self.jump(target); } + self.fire_branch_event(vm, src_offset, branch_taken, target)?; Ok(None) } @@ -2897,11 +3146,13 @@ impl ExecutingFrame<'_> { fn execute_for_iter(&mut self, vm: &VirtualMachine, target: bytecode::Label) -> FrameResult { let top_of_stack = PyIter::new(self.top_value()); let next_obj = top_of_stack.next(vm); + let src_offset = (self.lasti() - 1) * 2; // Check the next object: match next_obj { Ok(PyIterReturn::Return(value)) => { self.push_value(value); + self.fire_branch_event(vm, src_offset, false, target)?; Ok(None) } Ok(PyIterReturn::StopIteration(_)) => { @@ -2925,6 +3176,7 @@ impl ExecutingFrame<'_> { target }; self.jump(jump_target); + self.fire_branch_event(vm, src_offset, true, jump_target)?; Ok(None) } Err(next_error) => { diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 53b05f477f5..1a36bd1fa1f 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -1,3 +1,5 @@ +pub(crate) mod monitoring; + use crate::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule, convert::ToPyObject}; #[cfg(all(not(feature = "host_env"), feature = "stdio"))] @@ -73,6 +75,9 @@ mod sys { RUST_MULTIARCH.replace("-unknown", "") } + #[pymodule(name = "monitoring", with(super::monitoring::sys_monitoring))] + pub(super) mod monitoring {} + #[pyclass(no_attr, name = "_BootstrapStderr")] #[derive(Debug, PyPayload)] pub(super) struct BootstrapStderr; diff --git a/crates/vm/src/stdlib/sys/monitoring.rs b/crates/vm/src/stdlib/sys/monitoring.rs new file mode 100644 index 00000000000..81a78d2fd46 --- /dev/null +++ b/crates/vm/src/stdlib/sys/monitoring.rs @@ -0,0 +1,872 @@ +use crate::{ + AsObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyCode, PyDictRef, PyNamespace, PyStrRef}, + function::FuncArgs, +}; +use crossbeam_utils::atomic::AtomicCell; +use std::collections::{HashMap, HashSet}; + +pub const TOOL_LIMIT: usize = 6; +const EVENTS_COUNT: usize = 19; +const LOCAL_EVENTS_COUNT: usize = 11; +const UNGROUPED_EVENTS_COUNT: usize = 18; + +// Event bit positions +bitflags::bitflags! { + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + pub struct MonitoringEvents: u32 { + const PY_START = 1 << 0; + const PY_RESUME = 1 << 1; + const PY_RETURN = 1 << 2; + const PY_YIELD = 1 << 3; + const CALL = 1 << 4; + const LINE = 1 << 5; + const INSTRUCTION = 1 << 6; + const JUMP = 1 << 7; + const BRANCH_LEFT = 1 << 8; + const BRANCH_RIGHT = 1 << 9; + const STOP_ITERATION = 1 << 10; + const RAISE = 1 << 11; + const EXCEPTION_HANDLED = 1 << 12; + const PY_UNWIND = 1 << 13; + const PY_THROW = 1 << 14; + const RERAISE = 1 << 15; + const C_RETURN = 1 << 16; + const C_RAISE = 1 << 17; + const BRANCH = 1 << 18; + } +} + +// Re-export as plain u32 constants for use in frame.rs hot-path checks +pub const EVENT_PY_START: u32 = MonitoringEvents::PY_START.bits(); +pub const EVENT_PY_RESUME: u32 = MonitoringEvents::PY_RESUME.bits(); +pub const EVENT_PY_RETURN: u32 = MonitoringEvents::PY_RETURN.bits(); +pub const EVENT_PY_YIELD: u32 = MonitoringEvents::PY_YIELD.bits(); +pub const EVENT_CALL: u32 = MonitoringEvents::CALL.bits(); +pub const EVENT_LINE: u32 = MonitoringEvents::LINE.bits(); +pub const EVENT_INSTRUCTION: u32 = MonitoringEvents::INSTRUCTION.bits(); +pub const EVENT_JUMP: u32 = MonitoringEvents::JUMP.bits(); +pub const EVENT_BRANCH_LEFT: u32 = MonitoringEvents::BRANCH_LEFT.bits(); +pub const EVENT_BRANCH_RIGHT: u32 = MonitoringEvents::BRANCH_RIGHT.bits(); +pub const EVENT_RAISE: u32 = MonitoringEvents::RAISE.bits(); +pub const EVENT_EXCEPTION_HANDLED: u32 = MonitoringEvents::EXCEPTION_HANDLED.bits(); +pub const EVENT_PY_UNWIND: u32 = MonitoringEvents::PY_UNWIND.bits(); +pub const EVENT_C_RETURN: u32 = MonitoringEvents::C_RETURN.bits(); +const EVENT_C_RAISE: u32 = MonitoringEvents::C_RAISE.bits(); +const EVENT_STOP_ITERATION: u32 = MonitoringEvents::STOP_ITERATION.bits(); +pub const EVENT_PY_THROW: u32 = MonitoringEvents::PY_THROW.bits(); +const EVENT_BRANCH: u32 = MonitoringEvents::BRANCH.bits(); +pub const EVENT_RERAISE: u32 = MonitoringEvents::RERAISE.bits(); +const EVENT_C_RETURN_MASK: u32 = EVENT_C_RETURN | EVENT_C_RAISE; + +const EVENT_NAMES: [&str; EVENTS_COUNT] = [ + "PY_START", + "PY_RESUME", + "PY_RETURN", + "PY_YIELD", + "CALL", + "LINE", + "INSTRUCTION", + "JUMP", + "BRANCH_LEFT", + "BRANCH_RIGHT", + "STOP_ITERATION", + "RAISE", + "EXCEPTION_HANDLED", + "PY_UNWIND", + "PY_THROW", + "RERAISE", + "C_RETURN", + "C_RAISE", + "BRANCH", +]; + +/// Interpreter-level monitoring state, shared by all threads. +pub struct MonitoringState { + pub tool_names: [Option; TOOL_LIMIT], + pub global_events: [u32; TOOL_LIMIT], + pub local_events: HashMap<(usize, usize), u32>, + pub callbacks: HashMap<(usize, usize), PyObjectRef>, + /// Per-instruction disabled tools: (code_id, offset, tool) + pub disabled: HashSet<(usize, usize, usize)>, + /// Cached MISSING sentinel singleton + pub missing: Option, + /// Cached DISABLE sentinel singleton + pub disable: Option, +} + +impl Default for MonitoringState { + fn default() -> Self { + Self { + tool_names: Default::default(), + global_events: [0; TOOL_LIMIT], + local_events: HashMap::new(), + callbacks: HashMap::new(), + disabled: HashSet::new(), + missing: None, + disable: None, + } + } +} + +impl MonitoringState { + /// Compute the OR of all tools' global_events + local_events. + /// This is used for the fast-path atomic mask to skip monitoring + /// when no events are registered at all. + pub fn combined_events(&self) -> u32 { + let global = self.global_events.iter().fold(0, |acc, &e| acc | e); + let local = self.local_events.values().fold(0, |acc, &e| acc | e); + global | local + } +} + +/// Global atomic mask: OR of all tools' events. Checked in the hot path +/// to skip monitoring overhead when no events are registered. +/// Lives in PyGlobalState alongside the PyMutex. +pub type MonitoringEventsMask = AtomicCell; + +/// Get the MISSING sentinel, creating it if necessary. +pub fn get_missing(vm: &VirtualMachine) -> PyObjectRef { + let mut state = vm.state.monitoring.lock(); + if let Some(ref m) = state.missing { + m.clone() + } else { + let m: PyObjectRef = sys_monitoring::MonitoringSentinel.into_ref(&vm.ctx).into(); + state.missing = Some(m.clone()); + m + } +} + +/// Get the DISABLE sentinel, creating it if necessary. +pub fn get_disable(vm: &VirtualMachine) -> PyObjectRef { + let mut state = vm.state.monitoring.lock(); + if let Some(ref d) = state.disable { + d.clone() + } else { + let d: PyObjectRef = sys_monitoring::MonitoringSentinel.into_ref(&vm.ctx).into(); + state.disable = Some(d.clone()); + d + } +} + +fn check_valid_tool(tool_id: i32, vm: &VirtualMachine) -> PyResult { + if !(0..TOOL_LIMIT as i32).contains(&tool_id) { + return Err(vm.new_value_error(format!("invalid tool {tool_id} (must be between 0 and 5)"))); + } + Ok(tool_id as usize) +} + +fn check_tool_in_use(tool: usize, vm: &VirtualMachine) -> PyResult<()> { + let state = vm.state.monitoring.lock(); + if state.tool_names[tool].is_some() { + Ok(()) + } else { + Err(vm.new_value_error(format!("tool {tool} is not in use"))) + } +} + +fn parse_single_event(event: i32, vm: &VirtualMachine) -> PyResult { + let event = u32::try_from(event) + .map_err(|_| vm.new_value_error("The callback can only be set for one event at a time"))?; + if event.count_ones() != 1 { + return Err(vm.new_value_error("The callback can only be set for one event at a time")); + } + let event_id = event.trailing_zeros() as usize; + if event_id >= EVENTS_COUNT { + return Err(vm.new_value_error(format!("invalid event {event}"))); + } + Ok(event_id) +} + +fn normalize_event_set(event_set: i32, local: bool, vm: &VirtualMachine) -> PyResult { + if event_set < 0 { + let kind = if local { + "local event set" + } else { + "event set" + }; + return Err(vm.new_value_error(format!("invalid {kind} 0x{event_set:x}"))); + } + + let mut event_set = event_set as u32; + if event_set >= (1 << EVENTS_COUNT) { + let kind = if local { + "local event set" + } else { + "event set" + }; + return Err(vm.new_value_error(format!("invalid {kind} 0x{event_set:x}"))); + } + + if (event_set & EVENT_C_RETURN_MASK) != 0 && (event_set & EVENT_CALL) != EVENT_CALL { + return Err(vm.new_value_error("cannot set C_RETURN or C_RAISE events independently")); + } + + event_set &= !EVENT_C_RETURN_MASK; + + if (event_set & EVENT_BRANCH) != 0 { + event_set &= !EVENT_BRANCH; + event_set |= EVENT_BRANCH_LEFT | EVENT_BRANCH_RIGHT; + } + + if local && event_set >= (1 << LOCAL_EVENTS_COUNT) { + return Err(vm.new_value_error(format!("invalid local event set 0x{event_set:x}"))); + } + + Ok(event_set) +} + +/// Update the global monitoring_events atomic mask from current state. +fn update_events_mask(vm: &VirtualMachine, state: &MonitoringState) { + vm.state.monitoring_events.store(state.combined_events()); +} + +fn use_tool_id(tool_id: i32, name: &str, vm: &VirtualMachine) -> PyResult<()> { + let tool = check_valid_tool(tool_id, vm)?; + let mut state = vm.state.monitoring.lock(); + if state.tool_names[tool].is_some() { + return Err(vm.new_value_error(format!("tool {tool_id} is already in use"))); + } + state.tool_names[tool] = Some(name.to_owned()); + Ok(()) +} + +fn clear_tool_id(tool_id: i32, vm: &VirtualMachine) -> PyResult<()> { + let tool = check_valid_tool(tool_id, vm)?; + let mut state = vm.state.monitoring.lock(); + if state.tool_names[tool].is_some() { + state.global_events[tool] = 0; + state + .local_events + .retain(|(local_tool, _), _| *local_tool != tool); + state.callbacks.retain(|(cb_tool, _), _| *cb_tool != tool); + state.disabled.retain(|&(_, _, t)| t != tool); + } + update_events_mask(vm, &state); + Ok(()) +} + +fn free_tool_id(tool_id: i32, vm: &VirtualMachine) -> PyResult<()> { + let tool = check_valid_tool(tool_id, vm)?; + let mut state = vm.state.monitoring.lock(); + if state.tool_names[tool].is_some() { + state.global_events[tool] = 0; + state + .local_events + .retain(|(local_tool, _), _| *local_tool != tool); + state.callbacks.retain(|(cb_tool, _), _| *cb_tool != tool); + state.disabled.retain(|&(_, _, t)| t != tool); + state.tool_names[tool] = None; + } + update_events_mask(vm, &state); + Ok(()) +} + +fn get_tool(tool_id: i32, vm: &VirtualMachine) -> PyResult> { + let tool = check_valid_tool(tool_id, vm)?; + let state = vm.state.monitoring.lock(); + Ok(state.tool_names[tool].clone()) +} + +fn register_callback( + tool_id: i32, + event: i32, + func: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + let tool = check_valid_tool(tool_id, vm)?; + let event_id = parse_single_event(event, vm)?; + + let mut state = vm.state.monitoring.lock(); + let prev = state + .callbacks + .remove(&(tool, event_id)) + .unwrap_or_else(|| vm.ctx.none()); + let branch_id = EVENT_BRANCH.trailing_zeros() as usize; + let branch_left_id = EVENT_BRANCH_LEFT.trailing_zeros() as usize; + let branch_right_id = EVENT_BRANCH_RIGHT.trailing_zeros() as usize; + if !vm.is_none(&func) { + state.callbacks.insert((tool, event_id), func.clone()); + // BRANCH is a composite event: also register for BRANCH_LEFT/RIGHT + if event_id == branch_id { + state.callbacks.insert((tool, branch_left_id), func.clone()); + state.callbacks.insert((tool, branch_right_id), func); + } + } else { + // Also clear BRANCH_LEFT/RIGHT when clearing BRANCH + if event_id == branch_id { + state.callbacks.remove(&(tool, branch_left_id)); + state.callbacks.remove(&(tool, branch_right_id)); + } + } + Ok(prev) +} + +fn get_events(tool_id: i32, vm: &VirtualMachine) -> PyResult { + let tool = check_valid_tool(tool_id, vm)?; + let state = vm.state.monitoring.lock(); + Ok(state.global_events[tool]) +} + +fn set_events(tool_id: i32, event_set: i32, vm: &VirtualMachine) -> PyResult<()> { + let tool = check_valid_tool(tool_id, vm)?; + check_tool_in_use(tool, vm)?; + let normalized = normalize_event_set(event_set, false, vm)?; + let mut state = vm.state.monitoring.lock(); + state.global_events[tool] = normalized; + update_events_mask(vm, &state); + Ok(()) +} + +fn get_local_events(tool_id: i32, code: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if code.downcast_ref::().is_none() { + return Err(vm.new_type_error("code must be a code object")); + } + let tool = check_valid_tool(tool_id, vm)?; + let code_id = code.get_id(); + let state = vm.state.monitoring.lock(); + Ok(state + .local_events + .get(&(tool, code_id)) + .copied() + .unwrap_or(0)) +} + +fn set_local_events( + tool_id: i32, + code: PyObjectRef, + event_set: i32, + vm: &VirtualMachine, +) -> PyResult<()> { + if code.downcast_ref::().is_none() { + return Err(vm.new_type_error("code must be a code object")); + } + let tool = check_valid_tool(tool_id, vm)?; + check_tool_in_use(tool, vm)?; + let normalized = normalize_event_set(event_set, true, vm)?; + let code_id = code.get_id(); + let mut state = vm.state.monitoring.lock(); + if normalized == 0 { + state.local_events.remove(&(tool, code_id)); + } else { + state.local_events.insert((tool, code_id), normalized); + } + update_events_mask(vm, &state); + Ok(()) +} + +fn restart_events(vm: &VirtualMachine) { + let mut state = vm.state.monitoring.lock(); + state.disabled.clear(); +} + +fn all_events(vm: &VirtualMachine) -> PyResult { + // Collect data under the lock, then release before calling into Python VM. + let masks: Vec<(&str, u8)> = { + let state = vm.state.monitoring.lock(); + EVENT_NAMES + .iter() + .take(UNGROUPED_EVENTS_COUNT) + .enumerate() + .filter_map(|(event_id, event_name)| { + let event_bit = 1u32 << event_id; + let mut tools_mask = 0u8; + for tool in 0..TOOL_LIMIT { + if (state.global_events[tool] & event_bit) != 0 { + tools_mask |= 1 << tool; + } + } + if tools_mask != 0 { + Some((*event_name, tools_mask)) + } else { + None + } + }) + .collect() + }; + let all_events = vm.ctx.new_dict(); + for (name, mask) in masks { + all_events.set_item(name, vm.ctx.new_int(mask).into(), vm)?; + } + Ok(all_events) +} + +// Event dispatch + +use core::cell::Cell; + +thread_local! { + /// Re-entrancy guard: prevents monitoring callbacks from triggering + /// additional monitoring events (which would cause infinite recursion). + static FIRING: Cell = const { Cell::new(false) }; + + /// Tracks whether a RERAISE event has been fired since the last + /// EXCEPTION_HANDLED. Used to suppress duplicate RERAISE from + /// cleanup handlers that chain through multiple exception table entries. + static RERAISE_PENDING: Cell = const { Cell::new(false) }; +} + +/// Fire an event for all tools that have the event bit set. +/// `cb_extra` contains the callback arguments after the code object. +fn fire( + vm: &VirtualMachine, + event: u32, + code: &PyRef, + offset: u32, + cb_extra: &[PyObjectRef], +) -> PyResult<()> { + // Prevent recursive event firing + if FIRING.with(|f| f.get()) { + return Ok(()); + } + + let event_id = event.trailing_zeros() as usize; + let code_id = code.get_id(); + + // C_RETURN and C_RAISE are implicitly enabled when CALL is set. + let check_bit = if event & EVENT_C_RETURN_MASK != 0 { + event | EVENT_CALL + } else { + event + }; + + // Collect callbacks and snapshot the DISABLE sentinel under a single lock. + let (callbacks, disable_sentinel): (Vec<(usize, PyObjectRef)>, Option) = { + let state = vm.state.monitoring.lock(); + let mut cbs = Vec::new(); + for tool in 0..TOOL_LIMIT { + let global = state.global_events[tool]; + let local = state + .local_events + .get(&(tool, code_id)) + .copied() + .unwrap_or(0); + if ((global | local) & check_bit) == 0 { + continue; + } + if state.disabled.contains(&(code_id, offset as usize, tool)) { + continue; + } + if let Some(cb) = state.callbacks.get(&(tool, event_id)) { + cbs.push((tool, cb.clone())); + } + } + (cbs, state.disable.clone()) + }; + + if callbacks.is_empty() { + return Ok(()); + } + + let mut args_vec = Vec::with_capacity(1 + cb_extra.len()); + args_vec.push(code.clone().into()); + args_vec.extend_from_slice(cb_extra); + let args = FuncArgs::from(args_vec); + + FIRING.with(|f| f.set(true)); + let result = (|| { + for (tool, cb) in callbacks { + let result = cb.call(args.clone(), vm)?; + if disable_sentinel.as_ref().is_some_and(|d| result.is(d)) { + // Only local events (event_id < LOCAL_EVENTS_COUNT) can be disabled. + // Non-local events (RAISE, EXCEPTION_HANDLED, PY_UNWIND, etc.) + // cannot be disabled per code object. + if event_id >= LOCAL_EVENTS_COUNT { + return Err(vm.new_value_error(format!( + "cannot disable {} events", + EVENT_NAMES[event_id] + ))); + } + let mut state = vm.state.monitoring.lock(); + state.disabled.insert((code_id, offset as usize, tool)); + } + } + Ok(()) + })(); + FIRING.with(|f| f.set(false)); + result +} + +// Public dispatch functions (called from frame.rs) + +pub fn fire_py_start(vm: &VirtualMachine, code: &PyRef, offset: u32) -> PyResult<()> { + fire( + vm, + EVENT_PY_START, + code, + offset, + &[vm.ctx.new_int(offset).into()], + ) +} + +pub fn fire_py_resume(vm: &VirtualMachine, code: &PyRef, offset: u32) -> PyResult<()> { + fire( + vm, + EVENT_PY_RESUME, + code, + offset, + &[vm.ctx.new_int(offset).into()], + ) +} + +pub fn fire_py_return( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + retval: &PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_PY_RETURN, + code, + offset, + &[vm.ctx.new_int(offset).into(), retval.clone()], + ) +} + +pub fn fire_py_yield( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + retval: &PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_PY_YIELD, + code, + offset, + &[vm.ctx.new_int(offset).into(), retval.clone()], + ) +} + +pub fn fire_call( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + callable: &PyObjectRef, + arg0: PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_CALL, + code, + offset, + &[vm.ctx.new_int(offset).into(), callable.clone(), arg0], + ) +} + +pub fn fire_c_return( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + callable: &PyObjectRef, + arg0: PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_C_RETURN, + code, + offset, + &[vm.ctx.new_int(offset).into(), callable.clone(), arg0], + ) +} + +pub fn fire_c_raise( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + callable: &PyObjectRef, + arg0: PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_C_RAISE, + code, + offset, + &[vm.ctx.new_int(offset).into(), callable.clone(), arg0], + ) +} + +pub fn fire_line( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + line: u32, +) -> PyResult<()> { + fire(vm, EVENT_LINE, code, offset, &[vm.ctx.new_int(line).into()]) +} + +pub fn fire_instruction(vm: &VirtualMachine, code: &PyRef, offset: u32) -> PyResult<()> { + fire( + vm, + EVENT_INSTRUCTION, + code, + offset, + &[vm.ctx.new_int(offset).into()], + ) +} + +pub fn fire_raise( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + exception: &PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_RAISE, + code, + offset, + &[vm.ctx.new_int(offset).into(), exception.clone()], + ) +} + +/// Only fires if no RERAISE has been fired since the last EXCEPTION_HANDLED, +/// preventing duplicate events from chained cleanup handlers. +pub fn fire_reraise( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + exception: &PyObjectRef, +) -> PyResult<()> { + if RERAISE_PENDING.with(|f| f.get()) { + return Ok(()); + } + RERAISE_PENDING.with(|f| f.set(true)); + fire( + vm, + EVENT_RERAISE, + code, + offset, + &[vm.ctx.new_int(offset).into(), exception.clone()], + ) +} + +pub fn fire_exception_handled( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + exception: &PyObjectRef, +) -> PyResult<()> { + RERAISE_PENDING.with(|f| f.set(false)); + fire( + vm, + EVENT_EXCEPTION_HANDLED, + code, + offset, + &[vm.ctx.new_int(offset).into(), exception.clone()], + ) +} + +pub fn fire_py_unwind( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + exception: &PyObjectRef, +) -> PyResult<()> { + RERAISE_PENDING.with(|f| f.set(false)); + fire( + vm, + EVENT_PY_UNWIND, + code, + offset, + &[vm.ctx.new_int(offset).into(), exception.clone()], + ) +} + +pub fn fire_py_throw( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + exception: &PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_PY_THROW, + code, + offset, + &[vm.ctx.new_int(offset).into(), exception.clone()], + ) +} + +#[allow(dead_code)] +pub fn fire_stop_iteration( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + exception: &PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_STOP_ITERATION, + code, + offset, + &[vm.ctx.new_int(offset).into(), exception.clone()], + ) +} + +pub fn fire_jump( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + destination: u32, +) -> PyResult<()> { + fire( + vm, + EVENT_JUMP, + code, + offset, + &[ + vm.ctx.new_int(offset).into(), + vm.ctx.new_int(destination).into(), + ], + ) +} + +pub fn fire_branch_left( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + destination: u32, +) -> PyResult<()> { + fire( + vm, + EVENT_BRANCH_LEFT, + code, + offset, + &[ + vm.ctx.new_int(offset).into(), + vm.ctx.new_int(destination).into(), + ], + ) +} + +pub fn fire_branch_right( + vm: &VirtualMachine, + code: &PyRef, + offset: u32, + destination: u32, +) -> PyResult<()> { + fire( + vm, + EVENT_BRANCH_RIGHT, + code, + offset, + &[ + vm.ctx.new_int(offset).into(), + vm.ctx.new_int(destination).into(), + ], + ) +} + +#[pymodule(sub)] +pub(super) mod sys_monitoring { + use super::*; + + #[pyclass(no_attr, module = "sys.monitoring", name = "_Sentinel")] + #[derive(Debug, PyPayload)] + pub(super) struct MonitoringSentinel; + + #[pyclass] + impl MonitoringSentinel {} + + #[pyattr(name = "DEBUGGER_ID")] + const DEBUGGER_ID: u8 = 0; + #[pyattr(name = "COVERAGE_ID")] + const COVERAGE_ID: u8 = 1; + #[pyattr(name = "PROFILER_ID")] + const PROFILER_ID: u8 = 2; + #[pyattr(name = "OPTIMIZER_ID")] + const OPTIMIZER_ID: u8 = 5; + + #[pyattr(once, name = "DISABLE")] + fn disable(vm: &VirtualMachine) -> PyObjectRef { + super::get_disable(vm) + } + + #[pyattr(once, name = "MISSING")] + fn missing(vm: &VirtualMachine) -> PyObjectRef { + super::get_missing(vm) + } + + #[pyattr(once)] + fn events(vm: &VirtualMachine) -> PyRef { + let events = PyNamespace::default().into_ref(&vm.ctx); + for (event_id, event_name) in EVENT_NAMES.iter().enumerate() { + events + .as_object() + .set_attr(*event_name, vm.ctx.new_int(1u32 << event_id), vm) + .expect("setting sys.monitoring.events attribute should not fail"); + } + events + .as_object() + .set_attr("NO_EVENTS", vm.ctx.new_int(0), vm) + .expect("setting sys.monitoring.events.NO_EVENTS should not fail"); + events + } + + #[pyfunction] + fn use_tool_id(tool_id: i32, name: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { + super::use_tool_id(tool_id, name.as_str(), vm) + } + + #[pyfunction] + fn clear_tool_id(tool_id: i32, vm: &VirtualMachine) -> PyResult<()> { + super::clear_tool_id(tool_id, vm) + } + + #[pyfunction] + fn free_tool_id(tool_id: i32, vm: &VirtualMachine) -> PyResult<()> { + super::free_tool_id(tool_id, vm) + } + + #[pyfunction] + fn get_tool(tool_id: i32, vm: &VirtualMachine) -> PyResult> { + super::get_tool(tool_id, vm) + } + + #[pyfunction] + fn register_callback( + tool_id: i32, + event: i32, + func: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + super::register_callback(tool_id, event, func, vm) + } + + #[pyfunction] + fn get_events(tool_id: i32, vm: &VirtualMachine) -> PyResult { + super::get_events(tool_id, vm) + } + + #[pyfunction] + fn set_events(tool_id: i32, event_set: i32, vm: &VirtualMachine) -> PyResult<()> { + super::set_events(tool_id, event_set, vm) + } + + #[pyfunction] + fn get_local_events(tool_id: i32, code: PyObjectRef, vm: &VirtualMachine) -> PyResult { + super::get_local_events(tool_id, code, vm) + } + + #[pyfunction] + fn set_local_events( + tool_id: i32, + code: PyObjectRef, + event_set: i32, + vm: &VirtualMachine, + ) -> PyResult<()> { + super::set_local_events(tool_id, code, event_set, vm) + } + + #[pyfunction] + fn restart_events(vm: &VirtualMachine) { + super::restart_events(vm) + } + + #[pyfunction] + fn _all_events(vm: &VirtualMachine) -> PyResult { + super::all_events(vm) + } +} diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index f6ce3448c03..cb99644ef0e 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -124,6 +124,8 @@ where thread_handles: parking_lot::Mutex::new(Vec::new()), #[cfg(feature = "threading")] shutdown_handles: parking_lot::Mutex::new(Vec::new()), + monitoring: PyMutex::default(), + monitoring_events: AtomicCell::new(0), }); // Create VM with the global state diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 3285885eea2..51b4c6c1362 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -155,6 +155,10 @@ pub struct PyGlobalState { /// Registry for non-daemon threads that need to be joined at shutdown #[cfg(feature = "threading")] pub shutdown_handles: parking_lot::Mutex>, + /// sys.monitoring state (tool names, events, callbacks) + pub monitoring: PyMutex, + /// Fast-path mask: OR of all tools' events. 0 means no monitoring overhead. + pub monitoring_events: stdlib::sys::monitoring::MonitoringEventsMask, } pub fn process_hash_secret_seed() -> u32 {