From efe7ea7293a7e878bf34c5fb9b5e6d253732aa38 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 16 Apr 2020 19:54:13 -0700 Subject: [PATCH] Minor modernization and readability improvement to the tokenizer example (GH-19558) (cherry picked from commit bf1a81258c0ecc8b52b9dcc53321c066b3ed4a67) Co-authored-by: Raymond Hettinger --- Doc/library/re.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 7c950bfd5b1fd5..9abbd8ba73616e 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1617,10 +1617,14 @@ The text categories are specified with regular expressions. The technique is to combine those into a single master regular expression and to loop over successive matches:: - import collections + from typing import NamedTuple import re - Token = collections.namedtuple('Token', ['type', 'value', 'line', 'column']) + class Token(NamedTuple): + type: str + value: str + line: int + column: int def tokenize(code): keywords = {'IF', 'THEN', 'ENDIF', 'FOR', 'NEXT', 'GOSUB', 'RETURN'}