8000 Rename fields in tokenizer mode to not be f-string specific · python/cpython@173e456 · GitHub
[go: up one dir, main page]
Skip to content

Commit 173e456

Browse files
lysnikolaoulysnikolaou
committed
Rename fields in tokenizer mode to not be f-string specific
1 parent a78c082 commit 173e456

File tree

4 files changed

+45
-45
lines changed

4 files changed

+45
-45
lines changed

‎Parser/lexer/buffer.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ _PyLexer_remember_fstring_buffers(struct tok_state *tok)
1313

1414
for (index = tok->tok_mode_stack_index; index >= 0; --index) {
1515
mode = &(tok->tok_mode_stack[index]);
16-
mode->f_string_start_offset = mode->f_string_start - tok->buf;
17-
mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf;
16+
mode->start_offset = mode->start - tok->buf;
17+
mode->multi_line_start_offset = mode->multi_line_start - tok->buf;
1818
}
1919
}
2020

@@ -27,8 +27,8 @@ _PyLexer_restore_fstring_buffers(struct tok_state *tok)
2727

2828
for (index = tok->tok_mode_stack_index; index >= 0; --index) {
2929
mode = &(tok->tok_mode_stack[index]);
30-
mode->f_string_start = tok->buf + mode->f_string_start_offset;
31-
mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset;
30+
mode->start = tok->buf + mode->start_offset;
31+
mode->multi_line_start = tok->buf + mode->multi_line_start_offset;
3232
}
3333
}
3434

‎Parser/lexer/lexer.c

Lines changed: 31 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
114114
assert(c == '}' || c == ':' || c == '!');
115115
tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
116116

117-
if (!(tok_mode->f_string_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
117+
if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
118118
return 0;
119119
}
120120
PyObject *res = NULL;
@@ -981,33 +981,33 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
981981
}
982982
tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok);
983983
the_current_tok->kind = TOK_FSTRING_MODE;
984-
the_current_tok->f_string_quote = quote;
985-
the_current_tok->f_string_quote_size = quote_size;
986-
the_current_tok->f_string_start = tok->start;
987-
the_current_tok->f_string_multi_line_start = tok->line_start;
988-
the_current_tok->f_string_line_start = tok->lineno;
989-
the_current_tok->f_string_start_offset = -1;
990-
the_current_tok->f_string_multi_line_start_offset = -1;
984+
the_current_tok->quote = quote;
985+
the_current_tok->quote_size = quote_size;
986+
the_current_tok->start = tok->start;
987+
the_current_tok->multi_line_start = tok->line_start;
988+
the_current_tok->first_line = tok->lineno;
989+
the_current_tok->start_offset = -1;
990+
the_current_tok->multi_line_start_offset = -1;
991991
the_current_tok->last_expr_buffer = NULL;
992992
the_current_tok->last_expr_size = 0;
993993
the_current_tok->last_expr_end = -1;
994994
the_current_tok->in_format_spec = 0;
995-
the_current_tok->f_string_debug = 0;
995+
the_current_tok->in_debug = 0;
996996

997997
enum string_kind_t string_kind = FSTRING;
998998
switch (*tok->start) {
999999
case 'T':
10001000
case 't':
1001-
the_current_tok->f_string_raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
1001+
the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
10021002
string_kind = TSTRING;
10031003
break;
10041004
case 'F':
10051005
case 'f':
1006-
the_current_tok->f_string_raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
1006+
the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
10071007
break;
10081008
case 'R':
10091009
case 'r':
1010-
the_current_tok->f_string_raw = 1;
1010+
the_current_tok->raw = 1;
10111011
if (Py_TOLOWER(*(tok->start + 1)) == 't') {
10121012
string_kind = TSTRING;
10131013
}
@@ -1079,8 +1079,8 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
10791079
* and if it is, then this must be a missing '}' token
10801080
* so raise the proper error */
10811081
tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
1082-
if (the_current_tok->f_string_quote == quote &&
1083-
the_current_tok->f_string_quote_size == quote_size) {
1082+
if (the_current_tok->quote == quote &&
1083+
the_current_tok->quote_size == quote_size) {
10841084
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
10851085
"%c-string: expecting '}'", TOK_GET_STRING_PREFIX(tok)));
10861086
}
@@ -1153,7 +1153,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
11531153
int cursor = current_tok->curly_bracket_depth - (c != '{');
11541154
int in_format_spec = current_tok->in_format_spec;
11551155
int cursor_in_format_with_debug =
1156-
cursor == 1 && (current_tok->f_string_debug || in_format_spec);
1156+
cursor == 1 && (current_tok->in_debug || in_format_spec);
11571157
int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
11581158
if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) {
11591159
return MAKE_TOKEN(ENDMARKER);
@@ -1261,7 +1261,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
12611261
current_tok->curly_bracket_expr_start_depth--;
12621262
current_tok->kind = TOK_FSTRING_MODE;
12631263
current_tok->in_format_spec = 0;
1264-
current_tok->f_string_debug = 0;
1264+
current_tok->in_debug = 0;
12651265
}
12661266
}
12671267
break;
@@ -1274,7 +1274,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
12741274
}
12751275

12761276
if( c == '=' && INSIDE_FSTRING_EXPR(current_tok)) {
1277-
current_tok->f_string_debug = 1;
1277+
current_tok->in_debug = 1;
12781278
}
12791279

12801280
/* Punctuation character */
@@ -1317,9 +1317,9 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
13171317
}
13181318

13191319
// Check if we are at the end of the string
1320-
for (int i = 0; i < current_tok->f_string_quote_size; i++) {
1320+
for (int i = 0; i < current_tok->quote_size; i++) {
13211321
int quote = tok_nextc(tok);
1322-
if (quote != current_tok->f_string_quote) {
1322+
if (quote != current_tok->quote) {
13231323
tok_backup(tok, quote);
13241324
goto f_string_middle;
13251325
}
@@ -1342,7 +1342,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
13421342
// TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle
13431343
// this.
13441344
tok->multi_line_start = tok->line_start;
1345-
while (end_quote_size != current_tok->f_string_quote_size) {
1345+
while (end_quote_size != current_tok->quote_size) {
13461346
int c = tok_nextc(tok);
13471347
if (tok->done == E_ERROR || tok->done == E_DECODE) {
13481348
return MAKE_TOKEN(ERRORTOKEN);
@@ -1353,7 +1353,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
13531353
INSIDE_FSTRING_EXPR(current_tok)
13541354
);
13551355

1356-
if (c == EOF || (current_tok->f_string_quote_size == 1 && c == '\n')) {
1356+
if (c == EOF || (current_tok->quote_size == 1 && c == '\n')) {
13571357
if (tok->decoding_erred) {
13581358
return MAKE_TOKEN(ERRORTOKEN);
13591359
}
@@ -1362,7 +1362,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
13621362
// it means that the format spec ends here and we should
13631363
// return to the regular mode.
13641364
if (in_format_spec && c == '\n') {
1365-
if (current_tok->f_string_quote_size == 1) {
1365+
if (current_tok->quote_size == 1) {
13661366
return MAKE_TOKEN(
13671367
_PyTokenizer_syntaxerror(
13681368
tok,
@@ -1382,15 +1382,15 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
13821382
// shift the tok_state's location into
13831383
// the start of string, and report the error
13841384
// from the initial quote character
1385-
tok->cur = (char *)current_tok->f_string_start;
1385+
tok->cur = (char *)current_tok->start;
13861386
tok->cur++;
1387-
tok->line_start = current_tok->f_string_multi_line_start;
1387+
tok->line_start = current_tok->multi_line_start;
13881388
int start = tok->lineno;
13891389

13901390
tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
1391-
tok->lineno = the_current_tok->f_string_line_start;
1391+
tok->lineno = the_current_tok->first_line;
13921392

1393-
if (current_tok->f_string_quote_size == 3) {
1393+
if (current_tok->quote_size == 3) {
13941394
_PyTokenizer_syntaxerror(tok,
13951395
"unterminated triple-quoted %c-string literal"
13961396
" (detected at line %d)",
@@ -1407,7 +1407,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
14071407
}
14081408
}
14091409

1410-
if (c == current_tok->f_string_quote) {
1410+
if (c == current_tok->quote) {
14111411
end_quote_size += 1;
14121412
continue;
14131413
} else {
@@ -1470,7 +1470,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
14701470
// brace. We have to restore and return the control back
14711471
// to the loop for the next iteration.
14721472
if (peek == '{' || peek == '}') {
1473-
if (!current_tok->f_string_raw) {
1473+
if (!current_tok->raw) {
14741474
if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) {
14751475
return MAKE_TOKEN(ERRORTOKEN);
14761476
}
@@ -1479,7 +1479,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
14791479
continue;
14801480
}
14811481

1482-
if (!current_tok->f_string_raw) {
1482+
if (!current_tok->raw) {
14831483
if (peek == 'N') {
14841484
/* Handle named unicode escapes (\N{BULLET}) */
14851485
peek = tok_nextc(tok);
@@ -1497,8 +1497,8 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
14971497

14981498
// Backup the f-string quotes to emit a final FSTRING_MIDDLE and
14991499
// add the quotes to the FSTRING_END in the next tokenizer iteration.
1500-
for (int i = 0; i < current_tok->f_string_quote_size; i++) {
1501-
tok_backup(tok, current_tok->f_string_quote);
1500+
for (int i = 0; i < current_tok->quote_size; i++) {
1501+
tok_backup(tok, current_tok->quote);
15021502
}
15031503
p_start = tok->start;
15041504
p_end = tok->cur;

‎Parser/lexer/state.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ _PyTokenizer_tok_new(void)
5454
tok->tok_extra_tokens = 0;
5555
tok->comment_newline = 0;
5656
tok->implicit_newline = 0;
57-
tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .f_string_quote='\0', .f_string_quote_size = 0, .f_string_debug=0};
57+
tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .quote='\0', .quote_size = 0, .in_debug=0};
5858
tok->tok_mode_stack_index = 0;
5959
#ifdef Py_DEBUG
6060
tok->debug = _Py_GetConfig()->parser_debug;

‎Parser/lexer/state.h

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -49,20 +49,20 @@ typedef struct _tokenizer_mode {
4949
int curly_bracket_depth;
5050
int curly_bracket_expr_start_depth;
5151

52-
char f_string_quote;
53-
int f_string_quote_size;
54-
int f_string_raw;
55-
const char* f_string_start;
56-
const char* f_string_multi_line_start;
57-
int f_string_line_start;
52+
char quote;
53+
int quote_size;
54+
int raw;
55+
const char* start;
56+
const char* multi_line_start;
57+
int first_line;
5858

59-
Py_ssize_t f_string_start_offset;
60-
Py_ssize_t f_string_multi_line_start_offset;
59+
Py_ssize_t start_offset;
60+
Py_ssize_t multi_line_start_offset;
6161

6262
Py_ssize_t last_expr_size;
6363
Py_ssize_t last_expr_end;
6464
char* last_expr_buffer;
65-
int f_string_debug;
65+
int in_debug;
6666
int in_format_spec;
6767

6868
enum string_kind_t string_kind;

0 commit comments

Comments
 (0)
0