Fix tab-based indentation tracking

marijnh · marijnh · commit 51af501d6aea · 2022-03-16T08:59:52.000+01:00
FIX: Fix a bug where indentation with tabs was tracked incorrectly, leading to spurious indent tokens and malformed trees. Issue codemirror/dev#770
diff --git a/src/tokens.js b/src/tokens.js
@@ -61,13 +61,20 @@ function IndentLevel(parent, depth) {
 
 const topIndent = new IndentLevel(null, 0)
 
+function countIndent(space) {
+  let depth = 0
+  for (let i = 0; i < space.length; i++)
+    depth += space.charCodeAt(i) == tab ? 8 - (depth % 8) : 1
+  return depth
+}
+
 export const trackIndent = new ContextTracker({
   start: topIndent,
   reduce(context, term) {
     return context.depth < 0 && bracketed.indexOf(term) > -1 ? context.parent : context
   },
   shift(context, term, stack, input) {
-    if (term == indent) return new IndentLevel(context, stack.pos - input.pos)
+    if (term == indent) return new IndentLevel(context, countIndent(input.read(input.pos, stack.pos)))
     if (term == dedent) return context.parent
     if (term == ParenL || term == BracketL || term == BraceL) return new IndentLevel(context, -1)
     return context
diff --git a/test/statement.txt b/test/statement.txt
@@ -329,3 +329,22 @@ def x():
 ==>
 
 Script(FunctionDefinition(def,VariableName,ParamList,Body(ExpressionStatement(VariableName),ExpressionStatement(VariableName))))
+
+# Can handle tab indentation
+
+class Employee:
+	first_name: str
+        last_name: str
+
+	def __init__(self, a):
+		self.first_name = a
+	  	self.last_name = a
+
+==>
+
+Script(ClassDefinition(class,VariableName,Body(
+  AssignStatement(VariableName,TypeDef(VariableName)),
+  AssignStatement(VariableName,TypeDef(VariableName)),
+  FunctionDefinition(def,VariableName,ParamList(VariableName,VariableName),Body(
+    AssignStatement(MemberExpression(VariableName,PropertyName),AssignOp,VariableName),
+    AssignStatement(MemberExpression(VariableName,PropertyName),AssignOp,VariableName))))))
