Fix bug in tokenizing empty lines

marijnh · marijnh · commit 5b3b5d200ad8 · 2021-08-09T10:46:17.000+02:00
diff --git a/src/tokens.js b/src/tokens.js
@@ -24,10 +24,10 @@ export const newlines = new ExternalTokenizer((input, stack) => {
     input.acceptToken(newlineBracketed, 1)
   } else {
     input.advance()
-    let space = 0
-    while (input.next == space || input.next == tab) { input.advance(); space++ }
+    let spaces = 0
+    while (input.next == space || input.next == tab) { input.advance(); spaces++ }
     let empty = input.next == newline || input.next == carriageReturn || input.next == hash
-    input.acceptToken(empty ? newlineEmpty : newlineToken, -space)
+    input.acceptToken(empty ? newlineEmpty : newlineToken, -spaces)
   }
 }, {contextual: true, fallback: true})
 
diff --git a/test/statement.txt b/test/statement.txt
@@ -318,3 +318,14 @@ self = True
 ==>
 
 Script(AssignStatement(VariableName,AssignOp,Boolean))
+
+# Trailing whitespace in block
+
+def x():
+  one
+  
+  two
+
+==>
+
+Script(FunctionDefinition(def,VariableName,ParamList,Body(ExpressionStatement(VariableName),ExpressionStatement(VariableName))))
