From e766306fa7c8b86a72b16a4f43927465ddcc4226 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 11 Jun 2021 18:17:00 +0100 Subject: [PATCH] bpo-44396: Update multi-line-start location when reallocating tokenizer buffers --- Lib/test/test_eof.py | 7 +++++++ .../2021-06-11-18-17-42.bpo-44396.Z9EKim.rst | 2 ++ Parser/tokenizer.c | 5 +++++ 3 files changed, 14 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-06-11-18-17-42.bpo-44396.Z9EKim.rst diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index 2d3b4ae4e591ef..abcbf046e2cc22 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -29,6 +29,13 @@ def test_EOFS(self): else: raise support.TestFailed + def test_EOFS_with_file(self): + expect = ("(, line 1)") + with os_helper.temp_dir() as temp_dir: + file_name = script_helper.make_script(temp_dir, 'foo', """'''this is \na \ntest""") + rc, out, err = script_helper.assert_python_failure(file_name) + self.assertIn(b'unterminated triple-quoted string literal (detected at line 3)', err) + def test_eof_with_line_continuation(self): expect = "unexpected EOF while parsing (, line 1)" try: diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-11-18-17-42.bpo-44396.Z9EKim.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-11-18-17-42.bpo-44396.Z9EKim.rst new file mode 100644 index 00000000000000..be72a7111dc8ae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-06-11-18-17-42.bpo-44396.Z9EKim.rst @@ -0,0 +1,2 @@ +Fix a possible crash in the tokenizer when raising syntax errors for +unclosed strings. Patch by Pablo Galindo. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 6002f3e05a890c..be9b13ebabb8e3 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -372,6 +372,8 @@ tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) if (newsize > tok->end - tok->buf) { char *newbuf = tok->buf; Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf; + Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf; + Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf; newbuf = (char *)PyMem_Realloc(newbuf, newsize); if (newbuf == NULL) { tok->done = E_NOMEM; @@ -382,6 +384,8 @@ tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) tok->inp = tok->buf + oldsize; tok->end = tok->buf + newsize; tok->start = start < 0 ? NULL : tok->buf + start; + tok->line_start = line_start < 0 ? NULL : tok->buf + line_start; + tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start; } return 1; } @@ -1883,6 +1887,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) while (end_quote_size != quote_size) { c = tok_nextc(tok); if (c == EOF || (quote_size == 1 && c == '\n')) { + assert(tok->multi_line_start != NULL); // shift the tok_state's location into // the start of string, and report the error // from the initial quote character