From fe3b8caf1d82e86504b7d2c603ef12a3d0e0689d Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend@python.org>
Date: Thu, 15 Feb 2024 10:05:44 +0100
Subject: [PATCH 1/7] Use a lexer to generate better error messages for invalid
 syntax

---
 Lib/test/test_clinic.py             |   6 +-
 Tools/clinic/clinic.py              | 120 ++++++++++++++++++----------
 Tools/clinic/libclinic/__init__.py  |   4 +
 Tools/clinic/libclinic/tokenizer.py |  32 ++++++++
 4 files changed, 119 insertions(+), 43 deletions(-)
 create mode 100644 Tools/clinic/libclinic/tokenizer.py

diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py
index f5e9b11ad1cc8a..2e71500c1a96ad 100644
--- a/Lib/test/test_clinic.py
+++ b/Lib/test/test_clinic.py
@@ -600,7 +600,7 @@ def test_no_c_basename_cloned(self):
             foo as = foo2
             [clinic start generated code]*/
         """
-        err = "No C basename provided after 'as' keyword"
+        err = "No C basename provided for 'foo' after 'as' keyword"
         self.expect_failure(block, err, lineno=5)
 
     def test_cloned_with_custom_c_basename(self):
@@ -1541,7 +1541,7 @@ def test_illegal_module_line(self):
             foo.bar => int
                 /
         """
-        err = "Illegal function name: 'foo.bar => int'"
+        err = "Invalid syntax: 'foo.bar => int'"
         self.expect_failure(block, err)
 
     def test_illegal_c_basename(self):
@@ -1555,7 +1555,7 @@ def test_illegal_c_basename(self):
 
     def test_no_c_basename(self):
         block = "foo as "
-        err = "No C basename provided after 'as' keyword"
+        err = "No C basename provided for 'foo' after 'as' keyword"
         self.expect_failure(block, err, strip=False)
 
     def test_single_star(self):
diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py
index 5d2617b3bd579f..c7130e834a9028 100755
--- a/Tools/clinic/clinic.py
+++ b/Tools/clinic/clinic.py
@@ -39,7 +39,6 @@
     Any,
     Final,
     Literal,
-    NamedTuple,
     NoReturn,
     Protocol,
     TypeVar,
@@ -4757,11 +4756,6 @@ class ParamState(enum.IntEnum):
     RIGHT_SQUARE_AFTER = 6
 
 
-class FunctionNames(NamedTuple):
-    full_name: str
-    c_basename: str
-
-
 class DSLParser:
     function: Function | None
     state: StateKeeper
@@ -5047,25 +5041,6 @@ def state_dsl_start(self, line: str) -> None:
 
         self.next(self.state_modulename_name, line)
 
-    def parse_function_names(self, line: str) -> FunctionNames:
-        left, as_, right = line.partition(' as ')
-        full_name = left.strip()
-        c_basename = right.strip()
-        if as_ and not c_basename:
-            fail("No C basename provided after 'as' keyword")
-        if not c_basename:
-            fields = full_name.split(".")
-            if fields[-1] == '__new__':
-                fields.pop()
-            c_basename = "_".join(fields)
-        if not libclinic.is_legal_py_identifier(full_name):
-            fail(f"Illegal function name: {full_name!r}")
-        if not libclinic.is_legal_c_identifier(c_basename):
-            fail(f"Illegal C basename: {c_basename!r}")
-        names = FunctionNames(full_name=full_name, c_basename=c_basename)
-        self.normalize_function_kind(names.full_name)
-        return names
-
     def normalize_function_kind(self, fullname: str) -> None:
         # Fetch the method name and possibly class.
         fields = fullname.split('.')
@@ -5089,7 +5064,7 @@ def normalize_function_kind(self, fullname: str) -> None:
             self.kind = METHOD_INIT
 
     def resolve_return_converter(
-        self, full_name: str, forced_converter: str
+        self, full_name: str, forced_converter: str | None
     ) -> CReturnConverter:
         if forced_converter:
             if self.kind in {GETTER, SETTER}:
@@ -5115,8 +5090,12 @@ def resolve_return_converter(
             return init_return_converter()
         return CReturnConverter()
 
-    def parse_cloned_function(self, names: FunctionNames, existing: str) -> None:
-        full_name, c_basename = names
+    def parse_cloned_function(
+        self,
+        full_name: str,
+        c_basename: str,
+        existing: str
+    ) -> None:
         fields = [x.strip() for x in existing.split('.')]
         function_name = fields.pop()
         module, cls = self.clinic._module_and_class(fields)
@@ -5158,6 +5137,72 @@ def parse_cloned_function(self, names: FunctionNames, existing: str) -> None:
         (cls or module).functions.append(function)
         self.next(self.state_function_docstring)
 
+    @staticmethod
+    def generate_c_basename(full_name: str) -> str:
+        fields = full_name.split(".")
+        if fields[-1] == '__new__':
+            fields.pop()
+        return "_".join(fields)
+
+    def parse_declaration(
+        self, line: str
+    ) -> tuple[str, str, str | None, str | None]:
+        full_name: str
+        c_basename: str = ""
+        cloned: str | None = None
+        returns: str | None = None
+
+        def invalid_syntax(msg: str | None = None) -> NoReturn:
+            preamble = "Invalid syntax"
+            if msg:
+                preamble += f" ({msg})"
+            fail(f"{preamble}: {line!r}\n\n"
+                 "Allowed syntax:\n"
+                 "[module.[submodule.]][class.]func [as c_name] [-> return_annotation]\n\n"
+                 "Nested submodules are allowed.")
+
+        # Parse line.
+        try:
+            tokens = libclinic.generate_tokens(line)
+        except ValueError as exc:
+            invalid_syntax(str(exc))
+
+        match [t.token for t in tokens]:
+            # Common cases.
+            case [full_name]: ...
+            case [full_name, "as", c_basename]: ...
+
+            # Cloning.
+            case [full_name, "=", cloned]: ...
+            case [full_name, "as", c_basename, "=", cloned]: ...
+            case [full_name, "="] | [full_name, "as", _, "="]:
+                fail(f"No source function provided for {full_name!r} after '=' keyword")
+
+            # With return annotation.
+            case [full_name, "as" | "=", _, "->"] | [full_name, "->"]:
+                fail(f"No return annotation provided for {full_name!r} after '->' keyword")
+            case [full_name, "->", *_]:
+                returns = line[tokens[2].pos:].strip()
+            case [full_name, "as", c_basename, "->", *_]:
+                returns = line[tokens[4].pos:].strip()
+
+            # Other cases of invalid syntax.
+            case [full_name, "as"] | [full_name, "as", "=" | "->", *_]:
+                fail(f"No C basename provided for {full_name!r} after 'as' keyword")
+            case _:
+                invalid_syntax()
+
+        # Validate input.
+        if not libclinic.is_legal_py_identifier(full_name):
+            fail(f"Illegal function name: {full_name!r}")
+        if not c_basename:
+            c_basename = self.generate_c_basename(full_name)
+        if not libclinic.is_legal_c_identifier(c_basename):
+            fail(f"Illegal C basename: {c_basename!r}")
+        self.normalize_function_kind(full_name)
+
+        return full_name, c_basename, cloned, returns
+
     def state_modulename_name(self, line: str) -> None:
         # looking for declaration, which establishes the leftmost column
         # line should be
@@ -5178,20 +5223,15 @@ def state_modulename_name(self, line: str) -> None:
         assert self.valid_line(line)
         self.indent.infer(line)
 
-        # are we cloning?
-        before, equals, existing = line.rpartition('=')
-        if equals:
-            existing = existing.strip()
-            if libclinic.is_legal_py_identifier(existing):
-                # we're cloning!
-                names = self.parse_function_names(before)
-                return self.parse_cloned_function(names, existing)
-
-        line, _, returns = line.partition('->')
-        returns = returns.strip()
-        full_name, c_basename = self.parse_function_names(line)
+        full_name, c_basename, cloned, returns = self.parse_declaration(line)
+
+        # Handle cloning.
+        if cloned and libclinic.is_legal_py_identifier(cloned):
+            return self.parse_cloned_function(full_name, c_basename, cloned)
+
         return_converter = self.resolve_return_converter(full_name, returns)
 
+        # Split out function name, and determine module and class.
         fields = [x.strip() for x in full_name.split('.')]
         function_name = fields.pop()
         module, cls = self.clinic._module_and_class(fields)
diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py
index 738864a48c08d3..cf3af5e374b489 100644
--- a/Tools/clinic/libclinic/__init__.py
+++ b/Tools/clinic/libclinic/__init__.py
@@ -21,6 +21,7 @@
     is_legal_c_identifier,
     is_legal_py_identifier,
 )
+from .tokenizer import generate_tokens
 from .utils import (
     FormatCounterFormatter,
     compute_checksum,
@@ -51,6 +52,9 @@
     "is_legal_c_identifier",
     "is_legal_py_identifier",
 
+    # Parsing helpers
+    "generate_tokens",
+
     # Utility functions
     "FormatCounterFormatter",
     "compute_checksum",
diff --git a/Tools/clinic/libclinic/tokenizer.py b/Tools/clinic/libclinic/tokenizer.py
new file mode 100644
index 00000000000000..e14d13c7e86f25
--- /dev/null
+++ b/Tools/clinic/libclinic/tokenizer.py
@@ -0,0 +1,32 @@
+import io
+import shlex
+import dataclasses as dc
+from typing import Any
+
+
+@dc.dataclass
+class Token:
+    line: str
+    lineno: int
+    pos: int
+    token: str
+
+
+class Lexer(shlex.shlex):
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.wordchars += "."
+
+
+def generate_tokens(line: str, lineno: int = 0) -> list[Token]:
+    buf = io.StringIO(line)
+    lexer = Lexer(buf)
+    tokens: list[Token] = []
+    for idx, token in enumerate(lexer):
+        if idx and tokens[-1].token == "-" and token == ">":
+            tokens.pop()
+            token = "->"
+        pos = buf.tell() - len(token) - 1
+        tokens.append(Token(line, lineno, pos, token))
+    return tokens

From c926abc21084a829cfe55767086f88744e41eaa4 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend@python.org>
Date: Fri, 16 Feb 2024 10:00:03 +0100
Subject: [PATCH 2/7] Extend test suite

---
 Lib/test/test_clinic.py | 102 ++++++++++++++++++++++++++++------------
 1 file changed, 73 insertions(+), 29 deletions(-)

diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py
index 2e71500c1a96ad..21b95cd5998322 100644
--- a/Lib/test/test_clinic.py
+++ b/Lib/test/test_clinic.py
@@ -591,18 +591,6 @@ class C "void *" ""
         err = "'__new__' must be a class method"
         self.expect_failure(block, err, lineno=7)
 
-    def test_no_c_basename_cloned(self):
-        block = """
-            /*[clinic input]
-            foo2
-            [clinic start generated code]*/
-            /*[clinic input]
-            foo as = foo2
-            [clinic start generated code]*/
-        """
-        err = "No C basename provided for 'foo' after 'as' keyword"
-        self.expect_failure(block, err, lineno=5)
-
     def test_cloned_with_custom_c_basename(self):
         raw = dedent("""
             /*[clinic input]
@@ -1203,6 +1191,33 @@ def test_return_converter(self):
         """)
         self.assertIsInstance(function.return_converter, clinic.int_return_converter)
 
+    def test_return_converter_with_custom_c_name(self):
+        function = self.parse_function("""
+            module os
+            bar as foo -> int
+        """)
+        self.assertIsInstance(function.return_converter, clinic.int_return_converter)
+        self.assertEqual(function.name, "bar")
+        self.assertEqual(function.c_basename, "foo")
+
+    def test_return_converter_with_arg(self):
+        function = self.parse_function("""
+            module os
+            os.stat -> int(py_default=None)
+        """)
+        self.assertIsInstance(function.return_converter, clinic.int_return_converter)
+        self.assertEqual(function.return_converter.py_default, None)
+
+    def test_return_converter_with_arg_and_custom_c_name(self):
+        function = self.parse_function("""
+            module os
+            bar as foo -> int(py_default=None)
+        """)
+        self.assertIsInstance(function.return_converter, clinic.int_return_converter)
+        self.assertEqual(function.return_converter.py_default, None)
+        self.assertEqual(function.name, "bar")
+        self.assertEqual(function.c_basename, "foo")
+
     def test_return_converter_invalid_syntax(self):
         block = """
             module os
@@ -1535,28 +1550,57 @@ class foo.Bar "unused" "notneeded"
         # but it *is* a parameter
         self.assertEqual(1, len(function.parameters))
 
-    def test_illegal_module_line(self):
-        block = """
-            module foo
-            foo.bar => int
-                /
-        """
-        err = "Invalid syntax: 'foo.bar => int'"
-        self.expect_failure(block, err)
+    def test_invalid_syntax(self):
+        err = "Invalid syntax"
+        for block in (
+            "foo => bar",
+            "foo = bar baz",
+            "a b c d",
+            "foo as baz = bar ->",
+            "foo as bar bar = baz",
+        ):
+            with self.subTest(block=block):
+                self.expect_failure(block, err)
 
     def test_illegal_c_basename(self):
-        block = """
-            module foo
-            foo.bar as 935
-                /
-        """
-        err = "Illegal C basename: '935'"
-        self.expect_failure(block, err)
+        err = "Illegal C basename"
+        for block in (
+            "foo as 935",
+            "foo as ''",
+            "foo as a.c",
+        ):
+            with self.subTest(block=block):
+                self.expect_failure(block, err)
+
+    def test_no_return_annotation(self):
+        err = "No return annotation provided"
+        for block in (
+            "foo ->",
+            "foo = bar ->",
+            "foo as bar ->",
+        ):
+            with self.subTest(block=block):
+                self.expect_failure(block, err)
+
+    def test_clone_no_source_function(self):
+        err = "No source function provided"
+        for block in (
+            "foo =",
+            "foo as bar =",
+        ):
+            with self.subTest(block=block):
+                self.expect_failure(block, err)
 
     def test_no_c_basename(self):
-        block = "foo as "
         err = "No C basename provided for 'foo' after 'as' keyword"
-        self.expect_failure(block, err, strip=False)
+        for block in (
+            "foo as",
+            "foo as ",
+            "foo as = clone",
+            "foo as -> int",
+        ):
+            with self.subTest(block=block):
+                self.expect_failure(block, err)
 
     def test_single_star(self):
         block = """

From 33c21e74167ee17a0e1569fb8c5692e54150b4c9 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend@python.org>
Date: Fri, 16 Feb 2024 11:31:55 +0100
Subject: [PATCH 3/7] Validate cloned name post parsing

---
 Lib/test/test_clinic.py | 9 +++++++++
 Tools/clinic/clinic.py  | 4 +++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py
index 21b95cd5998322..88cbdb33012b4c 100644
--- a/Lib/test/test_clinic.py
+++ b/Lib/test/test_clinic.py
@@ -1572,6 +1572,15 @@ def test_illegal_c_basename(self):
             with self.subTest(block=block):
                 self.expect_failure(block, err)
 
+    def test_illegal_cloned_name(self):
+        err = "Illegal source function name"
+        for block in (
+            "foo = 935",
+            "foo = ''",
+        ):
+            with self.subTest(block=block):
+                self.expect_failure(block, err)
+
     def test_no_return_annotation(self):
         err = "No return annotation provided"
         for block in (
diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py
index c7130e834a9028..c19b06ca375436 100755
--- a/Tools/clinic/clinic.py
+++ b/Tools/clinic/clinic.py
@@ -5199,6 +5199,8 @@ def invalid_syntax(msg: str | None = None) -> NoReturn:
             c_basename = self.generate_c_basename(full_name)
         if not libclinic.is_legal_c_identifier(c_basename):
             fail(f"Illegal C basename: {c_basename!r}")
+        if cloned is not None and not libclinic.is_legal_py_identifier(cloned):
+            fail(f"Illegal source function name: {cloned!r}")
         self.normalize_function_kind(full_name)
 
         return full_name, c_basename, cloned, returns
@@ -5226,7 +5228,7 @@ def state_modulename_name(self, line: str) -> None:
         full_name, c_basename, cloned, returns = self.parse_declaration(line)
 
         # Handle cloning.
-        if cloned and libclinic.is_legal_py_identifier(cloned):
+        if cloned:
             return self.parse_cloned_function(full_name, c_basename, cloned)
 
         return_converter = self.resolve_return_converter(full_name, returns)

From 60553b74850eb29fc5d973d092d77ba3434e8766 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend@python.org>
Date: Fri, 16 Feb 2024 11:32:40 +0100
Subject: [PATCH 4/7] Remove now obsoleted comment

---
 Tools/clinic/clinic.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py
index c19b06ca375436..31ecc4bba4076a 100755
--- a/Tools/clinic/clinic.py
+++ b/Tools/clinic/clinic.py
@@ -5227,7 +5227,6 @@ def state_modulename_name(self, line: str) -> None:
 
         full_name, c_basename, cloned, returns = self.parse_declaration(line)
 
-        # Handle cloning.
         if cloned:
             return self.parse_cloned_function(full_name, c_basename, cloned)
 

From 9b93771ffb5b17cb9840d341bb6efead62ccc000 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend@python.org>
Date: Fri, 16 Feb 2024 15:03:10 +0100
Subject: [PATCH 5/7] Use regex instead; compromise by not detecting some edge
 cases

---
 Lib/test/test_clinic.py             |  5 --
 Tools/clinic/clinic.py              | 80 ++++++++++++++---------------
 Tools/clinic/libclinic/__init__.py  |  4 --
 Tools/clinic/libclinic/tokenizer.py | 32 ------------
 4 files changed, 39 insertions(+), 82 deletions(-)
 delete mode 100644 Tools/clinic/libclinic/tokenizer.py

diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py
index 88cbdb33012b4c..7d6d5f7ff3ec59 100644
--- a/Lib/test/test_clinic.py
+++ b/Lib/test/test_clinic.py
@@ -1553,11 +1553,7 @@ class foo.Bar "unused" "notneeded"
     def test_invalid_syntax(self):
         err = "Invalid syntax"
         for block in (
-            "foo => bar",
-            "foo = bar baz",
-            "a b c d",
             "foo as baz = bar ->",
-            "foo as bar bar = baz",
         ):
             with self.subTest(block=block):
                 self.expect_failure(block, err)
@@ -1585,7 +1581,6 @@ def test_no_return_annotation(self):
         err = "No return annotation provided"
         for block in (
             "foo ->",
-            "foo = bar ->",
             "foo as bar ->",
         ):
             with self.subTest(block=block):
diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py
index 31ecc4bba4076a..e23330c5dce987 100755
--- a/Tools/clinic/clinic.py
+++ b/Tools/clinic/clinic.py
@@ -51,6 +51,12 @@
 import libclinic
 import libclinic.cpp
 from libclinic import ClinicError
+from libclinic.parser import (
+    RE_CLONE,
+    RE_C_BASENAME,
+    RE_FULLNAME,
+    RE_RETURNS,
+)
 
 
 # TODO:
@@ -5147,10 +5153,8 @@ def generate_c_basename(full_name: str) -> str:
     def parse_declaration(
         self, line: str
     ) -> tuple[str, str, str | None, str | None]:
-        full_name: str
-        c_basename: str = ""
-        cloned: str | None = None
-        returns: str | None = None
+        cloned = None
+        returns = None
 
         def invalid_syntax(msg: str | None = None) -> NoReturn:
             preamble = "Invalid syntax"
@@ -5161,48 +5165,42 @@ def invalid_syntax(msg: str | None = None) -> NoReturn:
                  "[module.[submodule.]][class.]func [as c_name] [-> return_annotation]\n\n"
                  "Nested submodules are allowed.")
 
-        # Parse line.
-        try:
-            tokens = libclinic.generate_tokens(line)
-        except ValueError as exc:
-            invalid_syntax(str(exc))
-
-        match [t.token for t in tokens]:
-            # Common cases.
-            case [full_name]: ...
-            case [full_name, "as", c_basename]: ...
-
-            # Cloning.
-            case [full_name, "=", cloned]: ...
-            case [full_name, "as", c_basename, "=", cloned]: ...
-            case [full_name, "="] | [full_name, "as", _, "="]:
-                fail(f"No source function provided for {full_name!r} after '=' keyword")
-
-            # With return annotation.
-            case [full_name, "as" | "=", _, "->"] | [full_name, "->"]:
-                fail(f"No return annotation provided for {full_name!r} after '->' keyword")
-            case [full_name, "->", *_]:
-                returns = line[tokens[2].pos:].strip()
-            case [full_name, "as", c_basename, "->", *_]:
-                returns = line[tokens[4].pos:].strip()
+        m = RE_FULLNAME.match(line)
+        assert m
+        full_name = m[1]
+        if not libclinic.is_legal_py_identifier(full_name):
+            fail(f"Illegal function name: {full_name!r}")
+        pos = m.end()
 
-            # Other cases of invalid syntax.
-            case [full_name, "as"] | [full_name, "as", "=" | "->", *_]:
+        m = RE_C_BASENAME.match(line, pos)
+        if m:
+            if not m[1]:
                 fail(f"No C basename provided for {full_name!r} after 'as' keyword")
-            case _:
+            c_basename = m[1]
+            if not libclinic.is_legal_c_identifier(c_basename):
+                fail(f"Illegal C basename: {c_basename!r}")
+            pos = m.end()
+        else:
+            c_basename = self.generate_c_basename(full_name)
+
+        m = RE_CLONE.match(line, pos)
+        if m:
+            if not m[1]:
+                fail(f"No source function provided for {full_name!r} after '=' keyword")
+            cloned = m[1]
+            if not libclinic.is_legal_py_identifier(cloned):
+                fail(f"Illegal source function name: {cloned!r}")
+            pos = m.end()
+
+        m = RE_RETURNS.match(line, pos)
+        if m:
+            if cloned:
                 invalid_syntax()
+            if not m[1]:
+                fail(f"No return annotation provided for {full_name!r} after '->' keyword")
+            returns = m[1].strip()
 
-        # Validate input.
-        if not libclinic.is_legal_py_identifier(full_name):
-            fail(f"Illegal function name: {full_name!r}")
-        if not c_basename:
-            c_basename = self.generate_c_basename(full_name)
-        if not libclinic.is_legal_c_identifier(c_basename):
-            fail(f"Illegal C basename: {c_basename!r}")
-        if cloned is not None and not libclinic.is_legal_py_identifier(cloned):
-            fail(f"Illegal source function name: {cloned!r}")
         self.normalize_function_kind(full_name)
-
         return full_name, c_basename, cloned, returns
 
     def state_modulename_name(self, line: str) -> None:
diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py
index cf3af5e374b489..738864a48c08d3 100644
--- a/Tools/clinic/libclinic/__init__.py
+++ b/Tools/clinic/libclinic/__init__.py
@@ -21,7 +21,6 @@
     is_legal_c_identifier,
     is_legal_py_identifier,
 )
-from .tokenizer import generate_tokens
 from .utils import (
     FormatCounterFormatter,
     compute_checksum,
@@ -52,9 +51,6 @@
     "is_legal_c_identifier",
     "is_legal_py_identifier",
 
-    # Parsing helpers
-    "generate_tokens",
-
     # Utility functions
     "FormatCounterFormatter",
     "compute_checksum",
diff --git a/Tools/clinic/libclinic/tokenizer.py b/Tools/clinic/libclinic/tokenizer.py
deleted file mode 100644
index e14d13c7e86f25..00000000000000
--- a/Tools/clinic/libclinic/tokenizer.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import io
-import shlex
-import dataclasses as dc
-from typing import Any
-
-
-@dc.dataclass
-class Token:
-    line: str
-    lineno: int
-    pos: int
-    token: str
-
-
-class Lexer(shlex.shlex):
-
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        super().__init__(*args, **kwargs)
-        self.wordchars += "."
-
-
-def generate_tokens(line: str, lineno: int = 0) -> list[Token]:
-    buf = io.StringIO(line)
-    lexer = Lexer(buf)
-    tokens: list[Token] = []
-    for idx, token in enumerate(lexer):
-        if idx and tokens[-1].token == "-" and token == ">":
-            tokens.pop()
-            token = "->"
-        pos = buf.tell() - len(token) - 1
-        tokens.append(Token(line, lineno, pos, token))
-    return tokens

From 1cc72489901880185b15213b77d9e1c868793c70 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend@python.org>
Date: Fri, 16 Feb 2024 15:07:18 +0100
Subject: [PATCH 6/7] Add parser.py

---
 Tools/clinic/libclinic/parser.py | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 Tools/clinic/libclinic/parser.py

diff --git a/Tools/clinic/libclinic/parser.py b/Tools/clinic/libclinic/parser.py
new file mode 100644
index 00000000000000..5f23047fff2d4b
--- /dev/null
+++ b/Tools/clinic/libclinic/parser.py
@@ -0,0 +1,7 @@
+import re
+
+
+RE_FULLNAME = re.compile(r"\s*([\w.]+)\s*")
+RE_C_BASENAME = re.compile(r"\bas\b\s*(?:([^-=\s]+)\s*)?")
+RE_CLONE = re.compile(r"=\s*(?:([^-=\s]+)\s*)?")
+RE_RETURNS = re.compile(r"->\s*(.*)")

From 3c83c5d2211d9d92ba19016f17d8dd79b7e34fe9 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend@python.org>
Date: Sat, 17 Feb 2024 00:02:42 +0100
Subject: [PATCH 7/7] Detect more cases of invalid syntax

---
 Lib/test/test_clinic.py | 3 +++
 Tools/clinic/clinic.py  | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py
index 7d6d5f7ff3ec59..39715bb26afa8e 100644
--- a/Lib/test/test_clinic.py
+++ b/Lib/test/test_clinic.py
@@ -1553,7 +1553,10 @@ class foo.Bar "unused" "notneeded"
     def test_invalid_syntax(self):
         err = "Invalid syntax"
         for block in (
+            "foo = bar baz",
+            "a b c d",
             "foo as baz = bar ->",
+            "foo as bar bar = baz",
         ):
             with self.subTest(block=block):
                 self.expect_failure(block, err)
diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py
index e23330c5dce987..d6c3af6c20b76d 100755
--- a/Tools/clinic/clinic.py
+++ b/Tools/clinic/clinic.py
@@ -5199,6 +5199,10 @@ def invalid_syntax(msg: str | None = None) -> NoReturn:
             if not m[1]:
                 fail(f"No return annotation provided for {full_name!r} after '->' keyword")
             returns = m[1].strip()
+            pos = m.end()
+
+        if pos != len(line):
+            invalid_syntax()
 
         self.normalize_function_kind(full_name)
         return full_name, c_basename, cloned, returns