8000 [3.12] gh-105259: Ensure we don't show newline characters for trailin… · python/cpython@67b288f · GitHub
[go: up one dir, main page]
Skip to content

Commit 67b288f

Browse files
miss-islingtonmiss-islington
authored
[3.12] gh-105259: Ensure we don't show newline characters for trailing NEWLINE tokens (GH-105364) (#105367)
1 parent 6f3a4fd commit 67b288f

File tree

5 files changed

+17
-5
lines changed

5 files changed

+17
-5
lines changed

‎Lib/test/test_tokenize.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1870,7 +1870,7 @@ def readline(encoding):
18701870
TokenInfo(type=NUMBER, string='1', start=(1, 0), end=(1, 1), line='1+1\n'),
18711871
TokenInfo(type=OP, string='+', start=(1, 1), end=(1, 2), line='1+1\n'),
18721872
TokenInfo(type=NUMBER, string='1', start=(1, 2), end=(1, 3), line='1+1\n'),
1873-
TokenInfo(type=NEWLINE, string='\n', start=(1, 3), end=(1, 4), line='1+1\n'),
1873+
TokenInfo(type=NEWLINE, string='', start=(1, 3), end=(1, 4), line='1+1\n'),
18741874
TokenInfo(type=ENDMARKER, string='', start=(2, 0), end=(2, 0), line='')
18751875
]
18761876
for 10000 encoding in ["utf-8", "latin-1", "utf-16"]:

‎Misc/NEWS.d/next/Core and Builtins/2023-06-06-11-37-53.gh-issue-105259.E2BGKL.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Don't include newline character for trailing ``NEWLINE`` tokens emitted in
2+
the :mod:`tokenize` module. Patch by Pablo Galindo

‎Parser/tokenizer.c

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ tok_new(void)
114114
tok->report_warnings = 1;
115115
tok->tok_extra_tokens = 0;
116116
tok->comment_newline = 0;
117+
tok->implicit_newline = 0;
117118
tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .f_string_quote='\0', .f_string_quote_size = 0, .f_string_debug=0};
118119
tok->tok_mode_stack_index = 0;
119120
tok->tok_report_warnings = 1;
@@ -355,10 +356,12 @@ tok_concatenate_interactive_new_line(struct tok_state *tok, const char *line) {
355356
return -1;
356357
}
357358
strcpy(new_str + current_size, line);
359+
tok->implicit_newline = 0;
358360
if (last_char != '\n') {
359361
/* Last line does not end in \n, fake one */
360362
new_str[current_size + line_size - 1] = '\n';
361363
new_str[current_size + line_size] = '\0';
364+
tok->implicit_newline = 1;
362365
}
363366
tok->interactive_src_start = new_str;
364367
tok->interactive_src_end = new_str + current_size + line_size;
@@ -1262,11 +1265,13 @@ tok_underflow_file(struct tok_state *tok) {
12621265
tok->done = E_EOF;
12631266
return 0;
12641267
}
1268+
tok->implicit_newline = 0;
12651269
if (tok->inp[-1] != '\n') {
12661270
assert(tok->inp + 1 < tok->end);
12671271
/* Last line does not end in \n, fake one */
12681272
*tok->inp++ = '\n';
12691273
*tok->inp = '\0';
1274+
tok->implicit_newline = 1;
12701275
}
12711276

12721277
ADVANCE_LINENO();
@@ -1304,11 +1309,13 @@ tok_underflow_readline(struct tok_state* tok) {
13041309
tok->done = E_EOF;
13051310
return 0;
13061311
}
1312+
tok->implicit_newline = 0;
13071313
if (tok->inp[-1] != '\n') {
13081314
assert(tok->inp + 1 < tok->end);
13091315
/* Last line does not end in \n, fake one */
13101316
*tok->inp++ = '\n';
13111317
*tok->inp = '\0';
1318+
tok->implicit_newline = 1;
13121319
}
13131320

13141321
ADVANCE_LINENO();

‎Parser/tokenizer.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ struct tok_state {
131131
int tok_report_warnings;
132132
int tok_extra_tokens;
133133
int comment_newline;
134+
int implicit_newline;
134135
#ifdef Py_DEBUG
135136
int debug;
136137
#endif

‎Python/Python-tokenize.c

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -243,10 +243,12 @@ tokenizeriter_next(tokenizeriterobject *it)
243243
}
244244
else if (type == NEWLINE) {
245245
Py_DECREF(str);
246-
if (it->tok->start[0] == '\r') {
247-
str = PyUnicode_FromString("\r\n");
248-
} else {
249-
str = PyUnicode_FromString("\n");
246+
if (!it->tok->implicit_newline) {
247+
if (it->tok->start[0] == '\r') {
248+
str = PyUnicode_FromString("\r\n");
249+
} else {
250+
str = PyUnicode_FromString("\n");
251+
}
250252
}
251253
end_col_offset++;
252254
}

0 commit comments

Comments
 (0)
0