8000 Get rid of calls to startOf · jrdek/python@877eaf4 · GitHub
[go: up one dir, main page]
Skip to content

Commit 877eaf4

Browse files
marijnhmarijnh
committed
Get rid of calls to startOf
By storing bracket nesting depth in the context
1 parent dc7d6ed commit 877eaf4

File tree

2 files changed

+31
-12
lines changed

2 files changed

+31
-12
lines changed

‎src/python.grammar

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,10 @@ expressions { commaSep<"*" expression | test> }
6363

6464
ImportStatement {
6565
kw<"import"> dottedName (kw<"as"> VariableName)? |
66-
kw<"from"> (("." | "...")+ dottedName? | dottedName) kw<"import"> ("*" | "(" importedNames ")" | importedNames)
66+
kw<"from"> (("." | "...")+ dottedName? | dottedName) kw<"import"> ("*" | importList | importedNames)
6767
}
6868
importedNames { commaSep<VariableName | VariableName kw<"as"> VariableName> }
69+
importList[@export] { "(" importedNames ")" }
6970

7071
commaSep<expr> { expr ("," expr)* ","? }
7172

@@ -140,7 +141,7 @@ expression[@isGroup=Expression] {
140141
SetExpression |
141142
SetComprehensionExpression |
142143
CallExpression { expression !trail ArgList } |
143-
MemberExpression { expression !trail ("[" commaSep<test | test? ":" test? (":" test?)?> "]" | "." PropertyName) } |
144+
MemberExpression { expression !trail (subscript | "." PropertyName) } |
144145
VariableName |
145146
Number |
146147
String | FormatString |
@@ -150,6 +151,10 @@ expression[@isGroup=Expression] {
150151
@specialize[@name=Boolean]<identifier, "True" | "False">
151152
}
152153

154+
subscript[@export] {
155+
"[" commaSep<test | test? ":" test? (":" test?)?> "]"
156+
}
157+
153158
ParenthesizedExpression { "(" (testNamed | "*" expression | YieldExpression) ")" }
154159

155160
TupleExpression { "(" (testNamed | "*" expression) (("," (testNamed | "*" expression))+ ","? | ",") ")" }
@@ -295,7 +300,9 @@ FormatReplacement { "{" (YieldExpression | commaSep<"*"? test>) FormatConversion
295300

296301
"..."[@name=Ellipsis]
297302

298-
"(" ")" "[" "]" "{" "}"
303+
"("[@export=ParenL] ")"
304+
"["[@export=BracketL] "]"
305+
"{"[@export=BraceL] "}"
299306

300307
"." "," ";" ":" "@" "*" "**"
301308

‎src/tokens.js

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,25 @@ import {ExternalTokenizer, ContextTracker} from "@lezer/lr"
22
import {
33
newline as newlineToken, eof, newlineEmpty, newlineBracketed, indent, dedent, printKeyword,
44
ParenthesizedExpression, TupleExpression, ComprehensionExpression,
5-
ArrayExpression, ArrayComprehensionExpression, ArgList, ParamList,
6-
DictionaryExpression, DictionaryComprehensionExpression, SetExpression, SetComprehensionExpression
5+
ArrayExpression, ArrayComprehensionExpression, ArgList, ParamList, importList, subscript,
6+
DictionaryExpression, DictionaryComprehensionExpression, SetExpression, SetComprehensionExpression, FormatReplacement,
7+
ParenL, BraceL, BracketL
78
} from "./parser.terms.js"
89

910
const newline = 10, carriageReturn = 13, space = 32, tab = 9, hash = 35, parenOpen = 40, dot = 46
1011

1112
const bracketed = [
12-
ParenthesizedExpression, TupleExpression, ComprehensionExpression, ArrayExpression, ArrayComprehensionExpression,
13-
DictionaryExpression, DictionaryComprehensionExpression, SetExpression, SetComprehensionExpression, ArgList, ParamList
13+
ParenthesizedExpression, TupleExpression, ComprehensionExpression, importList, ArgList, ParamList,
14+
ArrayExpression, ArrayComprehensionExpression, subscript,
15+
SetExpression, SetComprehensionExpression,
16+
DictionaryExpression, DictionaryComprehensionExpression, FormatReplacement
1417
]
1518

1619
export const newlines = new ExternalTokenizer((input, stack) => {
1720
if (input.next < 0) {
1821
input.acceptToken(eof)
1922
} else if (input.next != newline && input.next != carriageReturn) {
20-
} else if (stack.startOf(bracketed) != null) {
23+
} else if (stack.context.depth < 0) {
2124
input.acceptToken(newlineBracketed, 1)
2225
} else {
2326
input.advance()
@@ -29,8 +32,10 @@ export const newlines = new ExternalTokenizer((input, stack) => {
2932
}, {contextual: true, fallback: true})
3033

3134
export const indentation = new ExternalTokenizer((input, stack) => {
35+
let cDepth = stack.context.depth
36+
if (cDepth < 0) return
3237
let prev = input.peek(-1), depth
33-
if ((prev == newline || prev == carriageReturn) && stack.startOf(bracketed) == null) {
38+
if ((prev == newline || prev == carriageReturn) && stack.context.depth >= 0) {
3439
let depth = 0, chars = 0
3540
for (;;) {
3641
if (input.next == space) depth++
@@ -39,16 +44,17 @@ export const indentation = new ExternalTokenizer((input, stack) => {
3944
input.advance()
4045
chars++
4146
}
42-
if (depth != stack.context.depth &&
47+
if (depth != cDepth &&
4348
input.next != newline && input.next != carriageReturn && input.next != hash) {
44-
if (depth < stack.context.depth) input.acceptToken(dedent, -chars)
49+
if (depth < cDepth) input.acceptToken(dedent, -chars)
4550
else input.acceptToken(indent)
4651
}
4752
}
4853
})
4954

5055
function IndentLevel(parent, depth) {
5156
this.parent = parent
57+
// -1 means this is not an actual indent level but a set of brackets
5258
this.depth = depth
5359
this.hash = (parent ? parent.hash + parent.hash << 8 : 0) + depth + (depth << 4)
5460
}
@@ -57,8 +63,14 @@ const topIndent = new IndentLevel(null, 0)
5763

5864
export const trackIndent = new ContextTracker({
5965
start: topIndent,
66+
reduce(context, term) {
67+
return context.depth < 0 && bracketed.indexOf(term) > -1 ? context.parent : context
68+
},
6069
shift(context, term, stack, input) {
61-
return term == indent ? new IndentLevel(context, stack.pos - input.pos) : term == dedent ? context.parent : context
70+
if (term == indent) return new IndentLevel(context, stack.pos - input.pos)
71+
if (term == dedent) return context.parent
72+
if (term == ParenL || term == BracketL || term == BraceL) return new IndentLevel(context, -1)
73+
return context
6274
},
6375
hash(context) { return context.hash }
6476
})

0 commit comments

Comments
 (0)
0