8000 gh-102856: Initial implementation of PEP 701 by pablogsal · Pull Request #102855 · python/cpython · GitHub
[go: up one dir, main page]
Skip to content

gh-102856: Initial implementation of PEP 701 #102855

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Apr 19, 2023
Merged

gh-102856: Initial implementation of PEP 701 #102855

Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
51e28e4
Initial implementation of PEP 701
pablogsal May 30, 2022
c2efa46
fixup! Remove double space in import error message (#103458)
pablogsal Apr 14, 2023
367d7f6
Fix ubsan: indexing into bracket_mark with index == -1
lysnikolaou Apr 17, 2023
349e8ff
Add macros to get the current mode
pablogsal Apr 17, 2023
29c5921
Fix nesting problem
pablogsal Apr 17, 2023
81d082b
📜🤖 Added by blurb_it.
blurb-it[bot] Apr 17, 2023
6d881ce
Update Parser/tokenizer.c
pablogsal Apr 18, 2023
6e77335
Update Parser/tokenizer.c
pablogsal Apr 18, 2023
2359fbf
Update Parser/tokenizer.c
pablogsal Apr 18, 2023
b0940fa
Update Parser/tokenizer.h
pablogsal Apr 18, 2023
afb310d
Update Parser/tokenizer.h
pablogsal Apr 18, 2023
13f942a
Fix whitespace to make patchcheck happy
lysnikolaou Apr 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add macros to get the current mode
  • Loading branch information
@pablogsal
pablogsal committed Apr 17, 2023
commit 349e8ff2a0589bf861d41d38c6644a7b56b9a144
50 changes: 36 additions & 14 deletions Parser/tokenizer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,28 @@
tok->lineno++; \
tok->col_offset = 0;

#ifdef Py_DEBUG
static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) {
assert(tok->tok_mode_stack_index >= 0);
assert(tok->tok_mode_stack_index < MAXLEVEL);
return &(tok->tok_mode_stack[tok->tok_mode_stack_index]);
}
static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) {
assert(tok->tok_mode_stack_index >= 0);
assert(tok->tok_mode_stack_index < MAXLEVEL);
return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]);
}
static inline int *TOK_GET_BRACKET_MARK(tokenizer_mode* mode) {
assert(mode->bracket_mark_index >= 0);
assert(mode->bracket_mark_index < MAX_EXPR_NEXTING);
return &(mode->bracket_mark[mode->bracket_mark_index]);
}
#else
#define TOK_GET_MODE(tok) (&(tok->tok_mode_stack[tok->tok_mode_stack_index]))
#define TOK_NEXT_MODE(tok) (&(tok->tok_mode_stack[++tok->tok_mode_stack_index]))
#define TOK_GET_BRACKET_MARK(mode) (&(mode->bracket_mark[mode->bracket_mark_index]))
#endif

/* Forward */
static struct tok_state *tok_new(void);
static int tok_nextc(struct tok_state *tok);
Expand Down Expand Up @@ -373,7 +395,7 @@ update_fstring_expr(struct tok_state *tok, char cur)
assert(tok->cur != NULL);

Py_ssize_t size = strlen(tok->cur);
tokenizer_mode *tok_mode = &(tok->tok_mode_stack[tok->tok_mode_stack_index]);
tokenizer_mode *tok_mode = TOK_GET_MODE(tok);

switch (cur) {
case '{':
Expand Down Expand Up @@ -2186,7 +2208,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t

p_start = tok->start;
p_end = tok->cur;
tokenizer_mode *current_tok = &(tok->tok_mode_stack[++tok->tok_mode_stack_index]);
tokenizer_mode *current_tok = TOK_NEXT_MODE(tok);
current_tok->kind = TOK_FSTRING_MODE;
current_tok->f_string_quote = quote;
current_tok->f_string_quote_size = quote_size;
Expand Down Expand Up @@ -2266,7 +2288,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
* does the initial quote matches with f-strings quotes
* and if it is, then this must be a missing '}' token
* so raise the proper error */
tokenizer_mode *current_tok = &(tok->tok_mode_stack[tok->tok_mode_stack_index]);
tokenizer_mode *current_tok = TOK_GET_MODE(tok);
if (current_tok->f_string_quote == quote &&
current_tok->f_string_quote_size == quote_size) {
return MAKE_TOKEN(syntaxerror(tok, "f-string: expecting '}'", start));
Expand Down Expand Up @@ -2318,7 +2340,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
/* Punctuation character */
int is_punctuation = (c == ':' || c == '}' || c == '!' || c == '{');
if (is_punctuation && tok->tok_mode_stack_index > 0 && current_tok->bracket_mark_index >= 0) {
int mark = current_tok->bracket_mark[current_tok->bracket_mark_index];
int mark = *TOK_GET_BRACKET_MARK(current_tok);
/* This code block gets executed before the bracket_stack is incremented
* by the `{` case, so for ensuring that we are on the 0th level, we need
* to adjust it manually */
Expand Down Expand Up @@ -2372,7 +2394,6 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
if (tok->tok_mode_stack_index > 0) {
current_tok->bracket_stack++;
}

break;
case ')':
case ']':
Expand All @@ -2397,7 +2418,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
if (tok->tok_mode_stack_index > 0 && opening == '{') {
assert(current_tok->bracket_stack >= 0);
int previous_bracket = current_tok->bracket_stack - 1;
if (previous_bracket == current_tok->bracket_mark[current_tok->bracket_mark_index]) {
if (previous_bracket == *TOK_GET_BRACKET_MARK(current_tok)) {
return MAKE_TOKEN(syntaxerror(tok, "f-string: unmatched '%c'", c));
}
}
Expand All @@ -2417,7 +2438,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t

if (tok->tok_mode_stack_index > 0) {
current_tok->bracket_stack--;
if (c == '}' && current_tok->bracket_stack == current_tok->bracket_mark[current_tok->bracket_mark_index]) {
if (c == '}' && current_tok->bracket_stack == *TOK_GET_BRACKET_MARK(current_tok)) {
current_tok->bracket_mark_index--;
current_tok->kind = TOK_FSTRING_MODE;
}
Expand Down Expand Up @@ -2461,10 +2482,10 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
if (current_tok->bracket_mark_index >= MAX_EXPR_NEXTING) {
return MAKE_TOKEN(syntaxerror(tok, "f-string: expressions nested too deeply"));
}

current_tok->bracket_mark[++current_tok->bracket_mark_index] = current_tok->bracket_stack;
current_tok->bracket_mark_index++;
*TOK_GET_BRACKET_MARK(current_tok) = current_tok->bracket_stack;
}
tok->tok_mode_stack[tok->tok_mode_stack_index].kind = TOK_REGULAR_MODE;
TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
return tok_get_normal_mode(tok, current_tok, token);
}

Expand Down Expand Up @@ -2529,8 +2550,9 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
if (peek != '{' || in_format_spec) {
tok_backup(tok, peek);
tok_backup(tok, c);
current_tok->bracket_mark[++current_tok->bracket_mark_index] = current_tok->bracket_stack;
tok->tok_mode_stack[tok->tok_mode_stack_index].kind = TOK_REGULAR_MODE;
current_tok->bracket_mark_index++;
*TOK_GET_BRACKET_MARK(current_tok) = current_tok->bracket_stack;
TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
p_start = tok->start;
p_end = tok->cur;
} else {
Expand All @@ -2556,7 +2578,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
} else {
tok_backup(tok, peek);
tok_backup(tok, c);
tok->tok_mode_stack[tok->tok_mode_stack_index].kind = TOK_REGULAR_MODE;
TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
p_start = tok->start;
p_end = tok->cur;
}
Expand Down Expand Up @@ -2606,7 +2628,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
static int
tok_get(struct tok_state *tok, struct token *token)
{
tokenizer_mode *current_tok = &(tok->tok_mode_stack[tok->tok_mode_stack_index]);
tokenizer_mode *current_tok = TOK_GET_MODE(tok);
if (current_tok->kind == TOK_REGULAR_MODE) {
return tok_get_normal_mode(tok, current_tok, token);
} else {
Expand Down
0