8000 bpo-29869: Allow underscores in numeric literals in lib2to3. (GH-1119) · python/cpython@a6e395d · GitHub
[go: up one dir, main page]
Skip to content
8000

Commit a6e395d

Browse files
nevsanMariatta
nevsan
authored and
Mariatta
committed
bpo-29869: Allow underscores in numeric literals in lib2to3. (GH-1119)
* Allow underscores in numeric literals in lib2to3. * Stricter literal parsing for Python 3.6 in lib2to3.pgen2.tokenize. * Add test case for underscores in literals in Python 3.
1 parent 873ef20 commit a6e395d

File tree

2 files changed

+30
-8
lines changed

2 files changed

+30
-8
lines changed

‎Lib/lib2to3/pgen2/tokenize.py

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -54,16 +54,16 @@ def maybe(*choices): return group(*choices) + '?'
5454
Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
5555
Name = r'[a-zA-Z_]\w*'
5656

57-
Binnumber = r'0[bB][01]*'
58-
Hexnumber = r'0[xX][\da-fA-F]*[lL]?'
59-
Octnumber = r'0[oO]?[0-7]*[lL]?'
60-
Decnumber = r'[1-9]\d*[lL]?'
57+
Binnumber = r'0[bB]_?[01]+(?:_[01]+)*'
58+
Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?'
59+
Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?'
60+
Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?')
6161
Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber)
62-
Exponent = r'[eE][-+]?\d+'
63-
Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
64-
Expfloat = r'\d+' + Exponent
62+
Exponent = r'[eE][-+]?\d+(?:_\d+)*'
63+
Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent)
64+
Expfloat = r'\d+(?:_\d+)*' + Exponent
6565
Floatnumber = group(Pointfloat, Expfloat)
66-
Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
66+
Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]')
6767
Number = group(Imagnumber, Floatnumber, Intnumber)
6868

6969
# Tail end of ' string.

‎Lib/lib2to3/tests/data/py3_test_grammar.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,28 @@ def testLongIntegers(self):
7272
x = 0b100000000000000000000000000000000000000000000000000000000000000000000
7373
x = 0B111111111111111111111111111111111111111111111111111111111111111111111
7474

75+
def testUnderscoresInNumbers(self):
76+
# Integers
77+
x = 1_0
78+
x = 123_456_7_89
79+
x = 0xabc_123_4_5
80+
x = 0X_abc_123
81+
x = 0B11_01
82+
x = 0b_11_01
83+
x = 0o45_67
84+
x = 0O_45_67
85+
86+
# Floats
87+
x = 3_1.4
88+
x = 03_1.4
89+
x = 3_1.
90+
x = .3_1
91+
x = 3.1_4
92+
x = 0_3.1_4
93+
x = 3e1_4
94+
x = 3_1e+4_1
95+
x = 3_1E-4_1
96+
7597
def testFloats(self):
7698
x = 3.14
7799
x = 314.

0 commit comments

Comments
 (0)
0