8000 refactor(compiler): capture `fullStart` locations when tokenizing (#3… · angular/angular@ff31b43 · GitHub
[go: up one dir, main page]

Skip to content

Commit ff31b43

Browse files
petebacondarwinmhevery
authored andcommitted
refactor(compiler): capture fullStart locations when tokenizing (#39589)
This commit ensures that when leading whitespace is skipped by the tokenizer, the original start location (before skipping) is captured in the `fullStart` property of the token's source-span. PR Close #39589
1 parent 1aee8b3 commit ff31b43

File tree

2 files changed

+27
-13
lines changed

2 files changed

+27
-13
lines changed

packages/compiler/src/ml_parser/lexer.ts

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -918,19 +918,20 @@ class PlainCharacterCursor implements CharacterCursor {
918918

919919
getSpan(start?: this, leadingTriviaCodePoints?: number[]): ParseSourceSpan {
920920
start = start || this;
921-
let cloned = false;
921+
let fullStart = start;
922922
if (leadingTriviaCodePoints) {
923923
while (this.diff(start) > 0 && leadingTriviaCodePoints.indexOf(start.peek()) !== -1) {
924-
if (!cloned) {
924+
if (fullStart === start) {
925925
start = start.clone() as this;
926-
cloned = true;
927926
}
928927
start.advance();
929928
}
930929
}
931-
return new ParseSourceSpan(
932-
new ParseLocation(start.file, start.state.offset, start.state.line, start.state.column),
933-
new ParseLocation(this.file, this.state.offset, this.state.line, this.state.column));
930+
const startLocation = this.locationFromCursor(start);
931+
const endLocation = this.locationFromCursor(this);
932+
const fullStartLocation =
933+
fullStart !== start ? this.locationFromCursor(fullStart) : startLocation;
934+
return new ParseSourceSpan(startLocation, endLocation, fullStartLocation);
934935
}
935936

936937
getChars(start: this): string {
@@ -960,6 +961,11 @@ class PlainCharacterCursor implements CharacterCursor {
960961
protected updatePeek(state: CursorState): void {
961962
state.peek = state.offset >= this.end ? chars.$EOF : this.charAt(state.offset);
962963
}
964+
965+
private locationFromCursor(cursor: this): ParseLocation {
966+
return new ParseLocation(
967+
cursor.file, cursor.state.offset, cursor.state.line, cursor.state.column);
968+
}
963969
}
964970

965971
class EscapedCharacterCursor extends PlainCharacterCursor {

packages/compiler/test/ml_parser/lexer_spec.ts

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -54,14 +54,14 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u
5454
});
5555

5656
it('should skip over leading trivia for source-span start', () => {
57-
expect(tokenizeAndHumanizeLineColumn(
58-
'<t>\n \t a</t>', {leadingTriviaChars: ['\n', ' ', '\t']}))
57+
expect(
58+
tokenizeAndHumanizeFullStart('<t>\n \t a</t>', {leadingTriviaChars: ['\n', ' ', '\t']}))
5959
.toEqual([
60-
[lex.TokenType.TAG_OPEN_START, '0:0'],
61-
[lex.TokenType.TAG_OPEN_END, '0:2'],
62-
[lex.TokenType.TEXT, '1:3'],
63-
[lex.TokenType.TAG_CLOSE, '1:4'],
64-
[lex.TokenType.EOF, '1:8'],
60+
[lex.TokenType.TAG_OPEN_START, '0:0', '0:0'],
61+
[lex.TokenType.TAG_OPEN_END, '0:2', '0:2'],
62+
[lex.TokenType.TEXT, '1:3', '0:3'],
63+
[lex.TokenType.TAG_CLOSE, '1:4', '1:4'],
64+
[lex.TokenType.EOF, '1:8', '1:8'],
6565
]);
6666
});
6767
});
@@ -1465,6 +1465,14 @@ function tokenizeAndHumanizeLineColumn(input: string, options?: lex.TokenizeOpti
14651465
.tokens.map(token => [<any>token.type, humanizeLineColumn(token.sourceSpan.start)]);
14661466
}
14671467

1468+
function tokenizeAndHumanizeFullStart(input: string, options?: lex.TokenizeOptions): any[] {
1469+
return tokenizeWithoutErrors(input, options)
1470+
.tokens.map(
1471+
token =>
1472+
[<any>token.type, humanizeLineColumn(token.sourceSpan.start),
1473+
humanizeLineColumn(token.sourceSpan.fullStart)]);
1474+
}
1475+
14681476
function tokenizeAndHumanizeErrors(input: string, options?: lex.TokenizeOptions): any[] {
14691477
return lex.tokenize(input, 'someUrl', getHtmlTagDefinition, options)
14701478
.errors.map(e => [<any>e.tokenType, e.msg, humanizeLineColumn(e.span.start)]);

0 commit comments

Comments
 (0)
0