8000 Move to new InputStream abstraction · geddski/python@0638dea · GitHub
[go: up one dir, main page]
Skip to content

Commit 0638dea

Browse files
marijnhmarijnh
committed
Move to new InputStream abstraction
1 parent 110a975 commit 0638dea

File tree

2 files changed

+41
-48
lines changed

2 files changed

+41
-48
lines changed

‎src/tokens.js

Lines changed: 38 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -13,44 +13,38 @@ const bracketed = [
1313
DictionaryExpression, DictionaryComprehensionExpression, SetExpression, SetComprehensionExpression, ArgList, ParamList
1414
]
1515

16-
let cachedIndent = 0, cachedInput = null, cachedPos = 0
17-
function getIndent(input, pos) {
18-
if (pos == cachedPos && input == cachedInput) return cachedIndent
19-
cachedInput = input; cachedPos = pos
20-
return cachedIndent = getIndentInner(input, pos)
21-
}
22-
23-
function getIndentInner(input, pos) {
24-
for (let indent = 0;; pos++) {
25-
let ch = input.get(pos)
26-
if (ch == space) indent++
27-
else if (ch == tab) indent += 8 - (indent % 8)
28-
else if (ch == newline || ch == carriageReturn || ch == hash) return -1
29-
else return indent
30-
}
31-
}
32-
33-
export const newlines = new ExternalTokenizer((input, token, stack) => {
34-
let next = input.get(token.start)
35-
if (next < 0) {
36-
token.accept(eof, token.start)
37-
} else if (next != newline && next != carriageReturn) {
16+
export const newlines = new ExternalTokenizer((input, stack) => {
17+
if (input.next < 0) {
18+
input.acceptToken(eof)
19+
} else if (input.next != newline && input.next != carriageReturn) {
3820
} else if (stack.startOf(bracketed) != null) {
39-
token.accept(newlineBracketed, token.start + 1)
40-
} else if (getIndent(input, token.start + 1) < 0) {
41-
token.accept(newlineEmpty, token.start + 1)
21+
input.acceptToken(newlineBracketed, 1)
4222
} else {
43-
token.accept(newlineToken, token.start + 1)
23+
input.advance()
24+
let space = 0
25+
while (input.next == space || input.next == tab) { input.advance(); space++ }
26+
let empty = input.next == newline || input.next == carriageReturn || input.next == hash
27+
input.acceptToken(empty ? newlineEmpty : newlineToken, -space)
4428
}
4529
}, {contextual: true, fallback: true})
4630

47-
export const indentation = new ExternalTokenizer((input, token, stack) => {
48-
let prev = input.get(token.start - 1), depth
49-
if ((prev == newline || prev == carriageReturn) &&
50-
(depth = getIndent(input, token.start)) >= 0 &&
51-
depth != stack.context.depth &&
52-
stack.startOf(bracketed) == null)
53-
token.accept(depth < stack.context.depth ? dedent : indent, token.start)
31+
export const indentation = new ExternalTokenizer((input, stack) => {
32+
let prev = input.peek(-1), depth
33+
if ((prev == newline || prev == carriageReturn) && stack.startOf(bracketed) == null) {
34+
let depth = 0, chars = 0
35+
for (;;) {
36+
if (input.next == space) depth++
37+
else if (input.next == tab) depth += 8 - (depth % 8)
38+
else break
39+
input.advance()
40+
chars++
41+
}
42+
if (depth != stack.context.depth &&
43+
input.next != newline && input.next != carriageReturn && input.next != hash) {
44+
if (depth < stack.context.depth) input.acceptToken(dedent, -chars)
45+
else input.acceptToken(indent)
46+
}
47+
}
5448
})
5549

5650
function IndentLevel(parent, depth) {
@@ -63,24 +57,23 @@ const topIndent = new IndentLevel(null, 0)
6357

6458
export const trackIndent = new ContextTracker({
6559
start: topIndent,
66-
shift(context, term, input, stack) {
67-
return term == indent ? new IndentLevel(context, getIndent(input, stack.pos)) :
68-
term == dedent ? context.parent : context
60+
shift(context, term, stack, input) {
61+
return term == indent ? new IndentLevel(context, stack.pos - input.pos) : term == dedent ? context.parent : context
6962
},
7063
hash(context) { return context.hash }
7164
})
7265

73-
export const legacyPrint = new ExternalTokenizer((input, token) => {
74-
let pos = token.start
75-
for (let print = "print", i = 0; i < print.length; i++, pos++)
76-
if (input.get(pos) != print.charCodeAt(i)) return
77-
let end = pos
78-
if (/\w/.test(String.fromCharCode(input.get(pos)))) return
79-
for (;; pos++) {
80-
let next = input.get(pos)
66+
export const legacyPrint = new ExternalTokenizer(input => {
67+
for (let i = 0; i < 5; i++) {
68+
if (input.next != "print".charCodeAt(i)) return
69+
input.advance()
70+
}
71+
if (/\w/.test(String.fromCharCode(input.next))) return
72+
for (let off = 0;; off++) {
73+
let next = input.peek(off)
8174
if (next == space || next == tab) continue
8275
if (next != parenOpen && next != dot && next != newline && next != carriageReturn && next != hash)
83-
token.accept(printKeyword, end)
76+
input.acceptToken(printKeyword)
8477
return
8578
}
8679
})

‎test/test-incremental.js

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import {parser} from "../dist/index.es.js"
22
import {fileTests} from "lezer-generator/dist/test"
3-
import {Tree, TreeFragment} from "lezer-tree"
3+
import {Tree, TreeFragment, parse} from "lezer-tree"
44

55
describe("Incremental parsing", () => {
66
// See https://github.com/codemirror/codemirror.next/issues/394
@@ -16,11 +16,11 @@ def b(self):
1616
""" ${"big block comment to fill up the reuse size quota\n ".repeat(150)} """
1717
pass
1818
`
19-
let ast = parser.parse(input)
19+
let ast = parse(parser, {input})
2020
let at = input.indexOf("pass")
2121
input = input.slice(0, at) + " " + input.slice(at)
2222
let cache = TreeFragment.applyChanges(TreeFragment.addTree(ast), [{fromA: at, toA: at, fromB: at, toB: at + 1}])
23-
let ast2 = parser.parse(input, 0, {fragments: cache})
23+
let ast2 = parse(parser, {input, fragments: cache})
2424
if (ast2.toString() != ast.toString()) throw new Error("Malformed tree")
2525

2626
let lastFunc = ast => {

0 commit comments

Comments
 (0)
0