8000 gh-119118: Fix performance regression in tokenize module by lysnikolaou · Pull Request #119615 · python/cpython · GitHub
[go: up one dir, main page]
Skip to content

gh-119118: Fix performance regression in tokenize module #119615

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
May 28, 2024
Merged

gh-119118: Fix performance regression in tokenize module #119615

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
66f9385
gh-119118: Fix performance regression in tokenize module
lysnikolaou May 20, 2024
143a420
📜🤖 Added by blurb_it.
blurb-it[bot] May 28, 2024
8318ab8
Update Python/Python-tokenize.c
lysnikolaou May 28, 2024
566adc3
WIP: strcmp lines (need to fix) and use line buffer for end col offset
lysnikolaou May 28, 2024
858eff1
Get rid of strcmp
lysnikolaou May 28, 2024
ab34370
Make NEWS linter happy
lysnikolaou May 28, 2024
b97c51e
Merge branch 'main' into performance-tokenize
pablogsal May 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
WIP: strcmp lines (need to fix) and use line buffer for end col offset
  • Loading branch information
@lysnikolaou
lysnikolaou committed May 28, 2024
commit 566adc3df02fd95b279d546518e9ad42b0e50503
25 changes: 25 additions & 0 deletions Parser/pegen.c
View file
Open in desktop
8000
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,31 @@ _PyPegen_interactive_exit(Parser *p)
return NULL;
}

Py_ssize_t
_PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset)
{
const char *data = PyUnicode_AsUTF8(line);

Py_ssize_t len = 0;
while (col_offset < end_col_offset) {
Py_UCS4 ch = data[col_offset];
if (ch < 0x80) {
col_offset += 1;
} else if ((ch & 0xe0) == 0xc0) {
col_offset += 2;
} else if ((ch & 0xf0) == 0xe0) {
col_offset += 3;
} else if ((ch & 0xf8) == 0xf0) {
col_offset += 4;
} else {
PyErr_SetString(PyExc_ValueError, "Invalid UTF-8 sequence");
return -1;
}
len++;
}
return len;
}

Py_ssize_t
_PyPegen_byte_offset_to_character_offset_raw(const char* str, Py_ssize_t col_offset)
{
Expand Down
1 change: 1 addition & 0 deletions Parser/pegen.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ int _PyPegen_fill_token(Parser *p);
expr_ty _PyPegen_name_token(Parser *p);
expr_ty _PyPegen_number_token(Parser *p);
void *_PyPegen_string_token(Parser *p);
Py_ssize_t _PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset); 8000
Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset);
Py_ssize_t _PyPegen_byte_offset_to_character_offset_raw(const char*, Py_ssize_t col_offset);

Expand Down
12 changes: 5 additions & 7 deletions Python/Python-tokenize.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ tokenizeriter_next(tokenizeriterobject *it)
size -= 1;
}

if (line_start != it->last_line_start || size != it->last_line_size) {
if (size != it->last_line_size || strcmp(line_start, it->last_line_start) != 0) {
// Line has changed since last token, so we fetch the new line and cache it
// in the iter object.
Py_XDECREF(it->last_line);
Expand All @@ -246,8 +246,9 @@ tokenizeriter_next(tokenizeriterobject *it)
Py_ssize_t end_lineno = it->tok->lineno;
Py_ssize_t col_offset = -1;
Py_ssize_t end_col_offset = -1;
Py_ssize_t byte_offset = -1;
if (token.start != NULL && token.start >= line_start) {
Py_ssize_t byte_offset = token.start - line_start;
byte_offset = token.start - line_start;
col_offset = byte_offset - it->byte_col_offset_diff;
}
if (token.end != NULL && token.end >= it->tok->line_start) {
Expand All @@ -256,12 +257,9 @@ tokenizeriter_next(tokenizeriterobject *it)
// If the whole token is at the same line, we can just use the token.start
// buffer for figuring out the new column offset, since using line is not
// performant for very long lines.
Py_ssize_t token_byte_offset = token.end - token.start;
Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_raw(
token.start, token_byte_offset
);
Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset);
end_col_offset = col_offset + token_col_offset;
it->byte_col_offset_diff += token_byte_offset - token_col_offset;
it->byte_col_offset_diff += token.end - token.start - token_col_offset;
} else {
end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset);
it->byte_col_offset_diff += end_byte_offset - end_col_offset;
Expand Down
0