bpo-40593: Improve syntax errors for invalid characters in source code.

python · serhiy-storchaka · May 12, 2020 · May 11, 2020 · May 11, 2020 · May 11, 2020
commit 8c069633d4fc403475b6168a09e3f4e04a9c1bc6
diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h
@@ -1222,6 +1222,8 @@ PyAPI_FUNC(void) _PyUnicode_ClearStaticStrings(void);
    and where the hash values are equal (i.e. a very probable match) */
 PyAPI_FUNC(int) _PyUnicode_EQ(PyObject *, PyObject *);
 
+PyAPI_FUNC(Py_ssize_t) _PyUnicode_ScanIdentifier(PyObject *);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/errcode.h b/Include/errcode.h
@@ -29,7 +29,6 @@ extern "C" {
 #define E_EOFS          23      /* EOF in triple-quoted string */
 #define E_EOLS          24      /* EOL in single-quoted string */
 #define E_LINECONT      25      /* Unexpected characters after a line continuation */
-#define E_IDENTIFIER    26      /* Invalid characters in identifier */
 #define E_BADSINGLE     27      /* Ill-formed single statement input */
 
 #ifdef __cplusplus

diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
@@ -583,7 +583,7 @@ def test_missing_expression(self):
                              ])
 
         # Different error message is raised for other whitespace characters.
-        self.assertAllRaise(SyntaxError, 'invalid character in identifier',
+        self.assertAllRaise(SyntaxError, r"invalid character '\u00a0' \(U\+00A0\)",
                             ["f'''{\xa0}'''",
                              "\xa0",
                              ])

diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py
@@ -57,6 +57,9 @@ def test_issue7820(self):
         # one byte in common with the UTF-16-LE BOM
         self.assertRaises(SyntaxError, eval, b'\xff\x20')
 
+        # one byte in common with the UTF-8 BOM
+        self.assertRaises(SyntaxError, eval, b'\xef\x20')
+
         # two bytes in common with the UTF-8 BOM
         self.assertRaises(SyntaxError, eval, b'\xef\xbb\x20')
 

diff --git a/Lib/test/test_unicode_identifiers.py b/Lib/test/test_unicode_identifiers.py
@@ -20,9 +20,11 @@ def test_non_bmp_normalized(self):
     def test_invalid(self):
         try:
             from test import badsyntax_3131
-        except SyntaxError as s:
-            self.assertEqual(str(s),
-              "invalid character in identifier (badsyntax_3131.py, line 2)")
+        except SyntaxError as err:
+            self.assertEqual(str(err),
+              "invalid character '€' (U+20AC) (badsyntax_3131.py, line 2)")
+            self.assertEqual(err.lineno, 2)
+            self.assertEqual(err.offset, 1)
         else:
             self.fail("expected exception didn't occur")
 

diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-05-11-13-50-52.bpo-40593.yuOXj3.rst b/Misc/NEWS.d/next/Core and Builtins/2020-05-11-13-50-52.bpo-40593.yuOXj3.rst
@@ -0,0 +1 @@
+Improved syntax errors for invalid characters in source code.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
@@ -12309,6 +12309,43 @@ unicode_isnumeric_impl(PyObject *self)
     Py_RETURN_TRUE;
 }
 
+Py_ssize_t
+_PyUnicode_ScanIdentifier(PyObject *self)
+{
+    Py_ssize_t i;
+    if (PyUnicode_READY(self) == -1)
+        return -1;
+
+    Py_ssize_t len = PyUnicode_GET_LENGTH(self);
+    if (len == 0) {
+        /* an empty string is not a valid identifier */
+        return 0;
+    }
+
+    int kind = PyUnicode_KIND(self);
+    const void *data = PyUnicode_DATA(self);
+    Py_UCS4 ch = PyUnicode_READ(kind, data, 0);
+    /* PEP 3131 says that the first character must be in
+       XID_Start and subsequent characters in XID_Continue,
+       and for the ASCII range, the 2.x rules apply (i.e
+       start with letters and underscore, continue with
+       letters, digits, underscore). However, given the current
+       definition of XID_Start and XID_Continue, it is sufficient
+       to check just for these, except that _ must be allowed
+       as starting an identifier.  */
+    if (!_PyUnicode_IsXidStart(ch) && ch != 0x5F /* LOW LINE */) {
+        return 0;
+    }
+
+    for (i = 1; i < len; i++) {
+        ch = PyUnicode_READ(kind, data, i);
+        if (!_PyUnicode_IsXidContinue(ch)) {
+            return i;
+        }
+    }
+    return i;
+}
+
 int
 PyUnicode_IsIdentifier(PyObject *self)
 {

diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c
@@ -337,9 +337,6 @@ tokenizer_error(Parser *p)
         case E_TOKEN:
             msg = "invalid token";
             break;
-        case E_IDENTIFIER:
-            msg = "invalid character in identifier";
-            break;
         case E_EOFS:
             RAISE_SYNTAX_ERROR("EOF while scanning triple-quoted string literal");
             return -1;

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
@@ -1101,25 +1101,48 @@ static int
 verify_identifier(struct tok_state *tok)
 {
     PyObject *s;
-    int result;
     if (tok->decoding_erred)
         return 0;
     s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
     if (s == NULL) {
         if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
-            PyErr_Clear();
-            tok->done = E_IDENTIFIER;
-        } else {
+            tok->done = E_DECODE;
+        }
+        else {
             tok->done = E_ERROR;
         }
         return 0;
     }
-    result = PyUnicode_IsIdentifier(s);
-    Py_DECREF(s);
-    if (result == 0) {
-        tok->done = E_IDENTIFIER;
+    Py_ssize_t invalid = _PyUnicode_ScanIdentifier(s);
+    if (invalid < 0) {
+        Py_DECREF(s);
+        tok->done = E_ERROR;
+        return 0;
     }
-    return result;
+    assert(PyUnicode_GET_LENGTH(s) > 0);
+    if (invalid < PyUnicode_GET_LENGTH(s)) {
+        Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
+        if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
+            /* Determine the offset in UTF-8 encoded input */
+            Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
+            if (s != NULL) {
+                Py_SETREF(s, PyUnicode_AsUTF8String(s));
+            }
+            if (s == NULL) {
+                tok->done = E_ERROR;
+                return 0;
+            }
+            tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s);
+        }
+        Py_DECREF(s);
+        // PyUnicode_FromFormatV() does not support %X
+        char hex[9];
+        snprintf(hex, sizeof(hex), "%04X", ch);
+        syntaxerror(tok, "invalid character '%c' (U+%s)", ch, hex);
+        return 0;
+    }
+    Py_DECREF(s);
+    return 1;
 }
 
 static int

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
@@ -1603,9 +1603,6 @@ err_input(perrdetail *err)
         msg = "unexpected character after line continuation character";
         break;
 
-    case E_IDENTIFIER:
-        msg = "invalid character in identifier";
-        break;
     case E_BADSINGLE:
         msg = "multiple statements found while compiling a single statement";
         break;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Improved syntax errors for invalid characters in source code.