8000 lexer: fix whitespace issues · go-python/gpython@6ab7061 · GitHub
[go: up one dir, main page]
Skip to content

Commit 6ab7061

Browse files
ncwncw
committed
lexer: fix whitespace issues
1 parent 8d5c74e commit 6ab7061

File tree

4 files changed

+8
-8
lines changed

4 files changed

+8
-8
lines changed

‎parser/grammar_data_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,7 @@ var grammarTestData = []struct {
203203
{"while True:\n pass\nelse:\n return\n", "exec", "Module(body=[While(test=NameConstant(value=True), body=[Pass()], orelse=[Return(value=None)])])"},
204204
{"if True: pass", "exec", "Module(body=[If(test=NameConstant(value=True), body=[Pass()], orelse=[])])"},
205205
{"if True:\n pass\n", "exec", "Module(body=[If(test=NameConstant(value=True), body=[Pass()], orelse=[])])"},
206+
{"if True:\n pass\n\n", "exec", "Module(body=[If(test=NameConstant(value=True), body=[Pass()], orelse=[])])"},
206207
{"if True:\n pass\n continue\nelse:\n break\n pass\n", "exec", "Module(body=[If(test=NameConstant(value=True), body=[Pass(), Continue()], orelse=[Break(), Pass()])])"},
207208
{"if a:\n continue\nelif b:\n break\nelif c:\n pass\nelif 10000 c:\n continue\n pass\n", "exec", "Module(body=[If(test=Name(id='a', ctx=Load()), body=[Continue()], orelse=[If(test=Name(id='b', ctx=Load()), body=[Break()], orelse=[If(test=Name(id='c', ctx=Load()), body=[Pass()], orelse=[If(test=Name(id='c', ctx=Load()), body=[Continue(), Pass()], orelse=[])])])])])"},
208209
{"if a:\n continue\nelif b:\n break\nelse:\n continue\n pass\n", "exec", "Module(body=[If(test=Name(id='a', ctx=Load()), body=[Continue()], orelse=[If(test=Name(id='b', ctx=Load()), body=[Break()], orelse=[Continue(), Pass()])])])"},

‎parser/lexer.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -372,12 +372,10 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
372372
x.line = trimmed
373373
x.state++
374374
case checkEmpty:
375+
despaced := strings.TrimSpace(x.line) // remove other whitespace other than " \t"
375376
// Ignore line if just white space or whitespace then comment
376-
if x.line == "" || x.line == "\n" || x.line[0] == '#' {
377+
if despaced == "" || despaced[0] == '#' {
377378
x.state = checkEof
378-
if x.line != "" {
379-
return NEWLINE
380-
}
381379
continue
382380
}
383381
x.state++

‎parser/lexer_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,6 @@ func TestLex(t *testing.T) {
204204
{"\n", "", "single", LexTokens{
205205
{SINGLE_INPUT, nil},
206206
{NEWLINE, nil},
207-
{NEWLINE, nil},
208207
}},
209208
{"pass", "", "single", LexTokens{
210209
{SINGLE_INPUT, nil},
@@ -218,9 +217,10 @@ func TestLex(t *testing.T) {
218217
}},
219218
{"\n#hello\n #comment\n", "", "exec", LexTokens{
220219
{FILE_INPUT, nil},
221-
{NEWLINE, nil},
222-
{NEWLINE, nil},
223-
{NEWLINE, nil},
220+
{ENDMARKER, nil},
221+
}},
222+
{"\n#hello\n\f #comment\n", "", "exec", LexTokens{
223+
{FILE_INPUT, nil},
224224
{ENDMARKER, nil},
225225
}},
226226
{"1\n 2\n", "", "eval", LexTokens{

‎parser/make_grammar_test.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,7 @@
232232
("while True:\n pass\nelse:\n return\n", "exec"),
233233
("if True: pass", "exec"),
234234
("if True:\n pass\n", "exec"),
235+
("if True:\n pass\n\n", "exec"),
235236
("""\
236237
if True:
237238
pass

0 commit comments

Comments
 (0)
0