From 05f943737c77279008b2e88c6181a8fb42f92453 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 12 Oct 2023 17:05:58 +0200 Subject: [PATCH 1/7] Allow the repl to show source code and complete tracebacks --- Include/internal/pycore_interp.h | 1 + Include/internal/pycore_parser.h | 12 +++++++++++- Lib/linecache.py | 10 ++++++++++ Lib/traceback.py | 13 +++++++++---- Parser/peg_api.c | 15 ++++++++++++++- Parser/pegen.c | 12 +++++++++++- Parser/pegen.h | 3 ++- 7 files changed, 58 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 80f16af54457fb..60d333ad7baa2e 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -233,6 +233,7 @@ struct _is { /* the initial PyInterpreterState.threads.head */ PyThreadState _initial_thread; + Py_ssize_t _interactive_src_count; }; diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index dd51b92801aebf..067b34c12c4e7f 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -58,7 +58,17 @@ extern struct _mod* _PyParser_ASTFromFile( PyCompilerFlags *flags, int *errcode, PyArena *arena); - +extern struct _mod* _PyParser_InteractiveASTFromFile( + FILE *fp, + PyObject *filename_ob, + const char *enc, + int mode, + const char *ps1, + const char *ps2, + PyCompilerFlags *flags, + int *errcode, + PyObject **interactive_src, + PyArena *arena); #ifdef __cplusplus } diff --git a/Lib/linecache.py b/Lib/linecache.py index 97644a8e3794e1..3be3ff00f95561 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -9,6 +9,7 @@ import sys import os import tokenize +import types __all__ = ["getline", "clearcache", "checkcache", "lazycache"] @@ -42,6 +43,8 @@ def getlines(filename, module_globals=None): if len(entry) != 1: return cache[filename][2] + if isinstance(filename, types.CodeType): + return [] try: return updatecache(filename, module_globals) except MemoryError: @@ -180,3 +183,10 @@ def lazycache(filename, module_globals): cache[filename] = (get_lines,) return True return False + +def _register_code(code, string, name): + cache[code] = ( + len(string), + None, + [line + '\n' for line in string.splitlines()], + name) diff --git a/Lib/traceback.py b/Lib/traceback.py index 12fcdad7dd4cb8..aa0da4fde150d0 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -331,7 +331,8 @@ def line(self): if self._line is None: if self.lineno is None: return None - self._line = linecache.getline(self.filename, self.lineno) + if self._line is None: + self._line = linecache.getline(self.filename, self.lineno) return self._line.strip() @@ -434,7 +435,6 @@ def _extract_from_extended_frame_gen(klass, frame_gen, *, limit=None, co = f.f_code filename = co.co_filename name = co.co_name - fnames.add(filename) linecache.lazycache(filename, f.f_globals) # Must defer line lookups until we have called checkcache. @@ -447,6 +447,7 @@ def _extract_from_extended_frame_gen(klass, frame_gen, *, limit=None, end_lineno=end_lineno, colno=colno, end_colno=end_colno)) for filename in fnames: linecache.checkcache(filename) + # If immediate lookup was desired, trigger lookups now. if lookup_lines: for f in result: @@ -479,8 +480,12 @@ def format_frame_summary(self, frame_summary): gets called for every frame to be printed in the stack summary. """ row = [] - row.append(' File "{}", line {}, in {}\n'.format( - frame_summary.filename, frame_summary.lineno, frame_summary.name)) + if frame_summary.filename.startswith("fp_interactive && tok->interactive_src_start && result && interactive_src != NULL) { + *interactive_src = PyUnicode_FromString(tok->interactive_src_start); + if (!interactive_src || _PyArena_AddPyObject(arena, *interactive_src) < 0) { + Py_XDECREF(interactive_src); + result = NULL; + goto error; + } + } + error: _PyTokenizer_Free(tok); return result; diff --git a/Parser/pegen.h b/Parser/pegen.h index 266d5219d45a9d..424f80acd7be3b 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -350,7 +350,8 @@ void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehensi Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *); void _PyPegen_Parser_Free(Parser *); mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *, - const char *, const char *, PyCompilerFlags *, int *, PyArena *); + const char *, const char *, PyCompilerFlags *, int *, PyObject **, + PyArena *); void *_PyPegen_run_parser(Parser *); mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *); asdl_stmt_seq *_PyPegen_interactive_exit(Parser *); From efdfaa4100567a848c2fd0351c50cf45e5a4bca6 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 13 Oct 2023 09:16:31 +0200 Subject: [PATCH 2/7] fixup! Allow the repl to show source code and complete tracebacks --- Lib/linecache.py | 3 --- Python/pythonrun.c | 55 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/Lib/linecache.py b/Lib/linecache.py index 3be3ff00f95561..c1c988d9df436c 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -9,7 +9,6 @@ import sys import os import tokenize -import types __all__ = ["getline", "clearcache", "checkcache", "lazycache"] @@ -43,8 +42,6 @@ def getlines(filename, module_globals=None): if len(entry) != 1: return cache[filename][2] - if isinstance(filename, types.CodeType): - return [] try: return updatecache(filename, module_globals) except MemoryError: diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 2e2747158c9ad4..110f585c607b35 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -40,7 +40,7 @@ /* Forward */ static void flush_io(void); static PyObject *run_mod(mod_ty, PyObject *, PyObject *, PyObject *, - PyCompilerFlags *, PyArena *); + PyCompilerFlags *, PyArena *, PyObject*); static PyObject *run_pyc_file(FILE *, PyObject *, PyObject *, PyCompilerFlags *); static int PyRun_InteractiveOneObjectEx(FILE *, PyObject *, PyCompilerFlags *); @@ -178,7 +178,8 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag // Call _PyParser_ASTFromFile() with sys.stdin.encoding, sys.ps1 and sys.ps2 static int pyrun_one_parse_ast(FILE *fp, PyObject *filename, - PyCompilerFlags *flags, PyArena *arena, mod_ty *pmod) + PyCompilerFlags *flags, PyArena *arena, + mod_ty *pmod, PyObject** interactive_src) { PyThreadState *tstate = _PyThreadState_GET(); @@ -236,9 +237,9 @@ pyrun_one_parse_ast(FILE *fp, PyObject *filename, } int errcode = 0; - *pmod = _PyParser_ASTFromFile(fp, filename, encoding, - Py_single_input, ps1, ps2, - flags, &errcode, arena); + *pmod = _PyParser_InteractiveASTFromFile(fp, filename, encoding, + Py_single_input, ps1, ps2, + flags, &errcode, interactive_src, arena); Py_XDECREF(ps1_obj); Py_XDECREF(ps2_obj); Py_XDECREF(encoding_obj); @@ -266,7 +267,8 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, } mod_ty mod; - int parse_res = pyrun_one_parse_ast(fp, filename, flags, arena, &mod); + PyObject *interactive_src; + int parse_res = pyrun_one_parse_ast(fp, filename, flags, arena, &mod, &interactive_src); if (parse_res != 0) { _PyArena_Free(arena); return parse_res; @@ -279,7 +281,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, } PyObject *main_dict = PyModule_GetDict(main_module); // borrowed ref - PyObject *res = run_mod(mod, filename, main_dict, main_dict, flags, arena); + PyObject *res = run_mod(mod, filename, main_dict, main_dict, flags, arena, interactive_src); _PyArena_Free(arena); Py_DECREF(main_module); if (res == NULL) { @@ -1149,7 +1151,7 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals, str, &_Py_STR(anon_string), start, flags, arena); if (mod != NULL) - ret = run_mod(mod, &_Py_STR(anon_string), globals, locals, flags, arena); + ret = run_mod(mod, &_Py_STR(anon_string), globals, locals, flags, arena, NULL); _PyArena_Free(arena); return ret; } @@ -1174,7 +1176,7 @@ pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *ret; if (mod != NULL) { - ret = run_mod(mod, filename, globals, locals, flags, arena); + ret = run_mod(mod, filename, globals, locals, flags, arena, NULL); } else { ret = NULL; @@ -1262,13 +1264,46 @@ run_eval_code_obj(PyThreadState *tstate, PyCodeObject *co, PyObject *globals, Py static PyObject * run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, - PyCompilerFlags *flags, PyArena *arena) + PyCompilerFlags *flags, PyArena *arena, PyObject* interactive_src) { PyThreadState *tstate = _PyThreadState_GET(); + PyObject* interactive_filename = NULL; + if (interactive_src) { + PyInterpreterState *interp = tstate->interp; + interactive_filename = PyUnicode_FromFormat("", interp->_interactive_src_count++); + // TODO: Maybe we are leaking + filename = interactive_filename; + } + PyCodeObject *co = _PyAST_Compile(mod, filename, flags, -1, arena); if (co == NULL) return NULL; + if (interactive_src) { + PyObject *linecache_module = PyImport_ImportModule("linecache"); + + if (linecache_module == NULL) { + return NULL; + } + + PyObject *print_tb_func = PyObject_GetAttrString(linecache_module, "_register_code"); + + if (print_tb_func == NULL || !PyCallable_Check(print_tb_func)) { + Py_DECREF(linecache_module); + return NULL; + } + + PyObject* result = PyObject_CallFunction(print_tb_func, "OOO", interactive_filename, interactive_src, filename); + if (!result) { + Py_DECREF(linecache_module); + Py_XDECREF(print_tb_func); + return NULL; + } + Py_DECREF(linecache_module); + Py_XDECREF(print_tb_func); + Py_DECREF(result); + } + if (_PySys_Audit(tstate, "exec", "O", co) < 0) { Py_DECREF(co); return NULL; From 757f3c022eb665f521fec8ee8fae480ddfef89bf Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 13 Oct 2023 09:21:37 +0200 Subject: [PATCH 3/7] fixup! fixup! Allow the repl to show source code and complete tracebacks --- .../2023-10-13-09-21-29.gh-issue-110805.vhU7A7.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-13-09-21-29.gh-issue-110805.vhU7A7.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-13-09-21-29.gh-issue-110805.vhU7A7.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-13-09-21-29.gh-issue-110805.vhU7A7.rst new file mode 100644 index 00000000000000..be90bb3564fd54 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-13-09-21-29.gh-issue-110805.vhU7A7.rst @@ -0,0 +1,2 @@ +Allow the repl to show source code and complete tracebacks. Patch by Pablo +Galindo From 171138451dde7fceb82d6c8875aa71c769507234 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 13 Oct 2023 10:16:54 +0200 Subject: [PATCH 4/7] fixup! fixup! fixup! Allow the repl to show source code and complete tracebacks --- Lib/test/test_cmd_line_script.py | 2 ++ Lib/test/test_repl.py | 62 ++++++++++++++++++++++++++++++++ Lib/traceback.py | 2 +- 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 1b588826010717..614c6b3c3b5299 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -203,6 +203,8 @@ def check_repl_stderr_flush(self, separate_stderr=False): stderr = p.stderr if separate_stderr else p.stdout self.assertIn(b'Traceback ', stderr.readline()) self.assertIn(b'File ""', stderr.readline()) + self.assertIn(b'1/0', stderr.readline()) + self.assertIn(b' ~^~', stderr.readline()) self.assertIn(b'ZeroDivisionError', stderr.readline()) def test_repl_stdout_flush(self): diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 58392f2384a3d9..720a784b1587eb 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -130,6 +130,68 @@ def test_close_stdin(self): output = process.communicate(user_input)[0] self.assertEqual(process.returncode, 0) self.assertIn('before close', output) + + def test_interactive_traceback_reporting(self): + user_input = "1 / 0 / 3 / 4" + p = spawn_repl() + p.stdin.write(user_input) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + + traceback_lines = output.splitlines()[-6:-1] + expected_lines = [ + "Traceback (most recent call last):", + " File \"\", line 1, in ", + " 1 / 0 / 3 / 4", + " ~~^~~", + "ZeroDivisionError: division by zero", + ] + self.assertEqual(traceback_lines, expected_lines) + + def test_interactive_traceback_reporting_multiple_input(self): + user_input1 = dedent(""" + def foo(x): + 1 / x + + """) + p = spawn_repl() + p.stdin.write(user_input1) + user_input2 = "foo(0)" + p.stdin.write(user_input2) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + + traceback_lines = output.splitlines()[-7:-1] + expected_lines = [ + ' File "", line 1, in ', + ' foo(0)', + ' File "", line 2, in foo', + ' 1 / x', + ' ~~^~~', + 'ZeroDivisionError: division by zero' + ] + self.assertEqual(traceback_lines, expected_lines) + + def test_interactive_source_is_in_linecache(self): + user_input = dedent(""" + def foo(x): + return x + 1 + + def bar(x): + return foo(x) + 2 + """) + p = spawn_repl() + p.stdin.write(user_input) + user_input2 = dedent(""" + import linecache + print(linecache.cache['']) + """) + p.stdin.write(user_input2) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + expected = "(30, None, [\'def foo(x):\\n\', \' return x + 1\\n\', \'\\n\'], \'\')" + self.assertIn(expected, output, expected) + class TestInteractiveModeSyntaxErrors(unittest.TestCase): diff --git a/Lib/traceback.py b/Lib/traceback.py index aa0da4fde150d0..e13f5f30cb81d0 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -481,7 +481,7 @@ def format_frame_summary(self, frame_summary): """ row = [] if frame_summary.filename.startswith("", line {}, in {}\n'.format( frame_summary.lineno, frame_summary.name)) else: row.append(' File "{}", line {}, in {}\n'.format( From 64e1c41a37f0f6a54b1fd66f28d744001e557291 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 13 Oct 2023 10:51:14 +0200 Subject: [PATCH 5/7] fixup! fixup! fixup! fixup! Allow the repl to show source code and complete tracebacks --- Lib/test/test_repl.py | 2 +- Lib/traceback.py | 3 +-- Python/pythonrun.c | 31 ++++++++++++++++++++++++------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 720a784b1587eb..062b5891b9a433 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -189,7 +189,7 @@ def bar(x): p.stdin.write(user_input2) output = kill_python(p) self.assertEqual(p.returncode, 0) - expected = "(30, None, [\'def foo(x):\\n\', \' return x + 1\\n\', \'\\n\'], \'\')" + expected = "(30, None, [\'def foo(x):\\n\', \' return x + 1\\n\', \'\\n\'], \'\')" self.assertIn(expected, output, expected) diff --git a/Lib/traceback.py b/Lib/traceback.py index e13f5f30cb81d0..7cc84b9c762aeb 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -331,8 +331,7 @@ def line(self): if self._line is None: if self.lineno is None: return None - if self._line is None: - self._line = linecache.getline(self.filename, self.lineno) + self._line = linecache.getline(self.filename, self.lineno) return self._line.strip() diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 110f585c607b35..0df76b8ac324de 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1267,33 +1267,50 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, PyCompilerFlags *flags, PyArena *arena, PyObject* interactive_src) { PyThreadState *tstate = _PyThreadState_GET(); - PyObject* interactive_filename = NULL; + PyObject* interactive_filename = filename; if (interactive_src) { PyInterpreterState *interp = tstate->interp; - interactive_filename = PyUnicode_FromFormat("", interp->_interactive_src_count++); - // TODO: Maybe we are leaking - filename = interactive_filename; + interactive_filename = PyUnicode_FromFormat( + "", interp->_interactive_src_count++ + ); + if (interactive_filename == NULL) { + return NULL; + } } - PyCodeObject *co = _PyAST_Compile(mod, filename, flags, -1, arena); - if (co == NULL) + PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, arena); + if (co == NULL) { + Py_DECREF(interactive_filename); return NULL; + } if (interactive_src) { PyObject *linecache_module = PyImport_ImportModule("linecache"); if (linecache_module == NULL) { + Py_DECREF(co); + Py_DECREF(interactive_filename); return NULL; } PyObject *print_tb_func = PyObject_GetAttrString(linecache_module, "_register_code"); if (print_tb_func == NULL || !PyCallable_Check(print_tb_func)) { + Py_DECREF(co); + Py_DECREF(interactive_filename); Py_DECREF(linecache_module); return NULL; } - PyObject* result = PyObject_CallFunction(print_tb_func, "OOO", interactive_filename, interactive_src, filename); + PyObject* result = PyObject_CallFunction( + print_tb_func, "OOO", + interactive_filename, + interactive_src, + filename + ); + + Py_DECREF(interactive_filename); + if (!result) { Py_DECREF(linecache_module); Py_XDECREF(print_tb_func); From 2da72a3dcd41e51baab354351d8055d46a310b8e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 13 Oct 2023 10:56:35 +0200 Subject: [PATCH 6/7] fixup! fixup! fixup! fixup! fixup! Allow the repl to show source code and complete tracebacks --- Python/pythonrun.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 0df76b8ac324de..347f84d665f178 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1295,13 +1295,22 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, PyObject *print_tb_func = PyObject_GetAttrString(linecache_module, "_register_code"); - if (print_tb_func == NULL || !PyCallable_Check(print_tb_func)) { + if (print_tb_func == NULL) { Py_DECREF(co); Py_DECREF(interactive_filename); Py_DECREF(linecache_module); return NULL; } + if (!PyCallable_Check(print_tb_func)) { + Py_DECREF(co); + Py_DECREF(interactive_filename); + Py_DECREF(linecache_module); + Py_DECREF(print_tb_func); + PyErr_SetString(PyExc_ValueError, "linecache._register_code is not callable"); + return NULL; + } + PyObject* result = PyObject_CallFunction( print_tb_func, "OOO", interactive_filename, @@ -1311,14 +1320,13 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, Py_DECREF(interactive_filename); + Py_DECREF(linecache_module); + Py_XDECREF(print_tb_func); + Py_XDECREF(result); if (!result) { - Py_DECREF(linecache_module); - Py_XDECREF(print_tb_func); + Py_DECREF(co); return NULL; } - Py_DECREF(linecache_module); - Py_XDECREF(print_tb_func); - Py_DECREF(result); } if (_PySys_Audit(tstate, "exec", "O", co) < 0) { From de39d7dc9966a9bec9203129aaf937997b8b1bb3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 13 Oct 2023 10:57:51 +0200 Subject: [PATCH 7/7] fixup! fixup! fixup! fixup! fixup! fixup! Allow the repl to show source code and complete tracebacks --- Lib/test/test_repl.py | 4 ++-- Python/pythonrun.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 062b5891b9a433..2ee5117bcd7bd4 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -130,7 +130,7 @@ def test_close_stdin(self): output = process.communicate(user_input)[0] self.assertEqual(process.returncode, 0) self.assertIn('before close', output) - + def test_interactive_traceback_reporting(self): user_input = "1 / 0 / 3 / 4" p = spawn_repl() @@ -176,7 +176,7 @@ def test_interactive_source_is_in_linecache(self): user_input = dedent(""" def foo(x): return x + 1 - + def bar(x): return foo(x) + 2 """) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 347f84d665f178..74994295a9b23a 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1313,8 +1313,8 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, PyObject* result = PyObject_CallFunction( print_tb_func, "OOO", - interactive_filename, - interactive_src, + interactive_filename, + interactive_src, filename );