8000 [3.12] gh-104719: IDLE - test existence of all tokenize references. (… · python/cpython@5e1799e · GitHub
[go: up one dir, main page]
Skip to content

Commit 5e1799e

Browse files
miss-islingtonterryjreedyJelleZijlstra
authored
[3.12] gh-104719: IDLE - test existence of all tokenize references. (GH-104767) (#104844)
gh-104719: IDLE - test existence of all tokenize references. (GH-104767) Class editor.IndentSearcher contains all editor references to tokenize module. Module io tokenize reference cover those other modules. (cherry picked from commit e561c09) Co-authored-by: Terry Jan Reedy <tjreedy@udel.edu> Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
1 parent b151660 commit 5e1799e

File tree

5 files changed

+54
-12
lines changed

5 files changed

+54
-12
lines changed

‎Lib/idlelib/NEWS.txt

+3
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ Released on 2023-10-02
44
=========================
55

66

7+
gh-104719: Remove IDLE's modification of tokenize.tabsize and test
8+
other uses of tokenize data and methods.
9+
710
gh-104499: Fix completions for Tk Aqua 8.7 (currently blank).
811

912
gh-104486: Make About print both tcl and tk versions if different,

‎Lib/idlelib/editor.py

+10-9
Original file line numberDiff line numberDiff line change
@@ -1571,7 +1571,7 @@ def reindent_to(self, column):
15711571
# blocks are found).
15721572

15731573
def guess_indent(self):
1574-
opener, indented = IndentSearcher(self.text, self.tabwidth).run()
1574+
opener, indented = IndentSearcher(self.text).run()
15751575
if opener and indented:
15761576
raw, indentsmall = get_line_indent(opener, self.tabwidth)
15771577
raw, indentlarge = get_line_indent(indented, self.tabwidth)
@@ -1609,15 +1609,10 @@ def get_line_indent(line, tabwidth):
16091609

16101610

16111611
class IndentSearcher:
1612+
"Manage initial indent guess, returned by run method."
16121613

1613-
# .run() chews over the Text widget, looking for a block opener
1614-
# and the stmt following it. Returns a pair,
1615-
# (line containing block opener, line containing stmt)
1616-
# Either or both may be None.
1617-
1618-
def __init__(self, text, tabwidth):
1614+
def __init__(self, text):
16191615
self.text = text
1620-
self.tabwidth = tabwidth
16211616
self.i = self.finished = 0
16221617
self.blkopenline = self.indentedline = None
16231618

@@ -1633,7 +1628,8 @@ def readline(self):
16331628
def tokeneater(self, type, token, start, end, line,
16341629
INDENT=tokenize.INDENT,
16351630
NAME=tokenize.NAME,
1636-
OPENERS=('class', 'def', 'for', 'if', 'try', 'while')):
1631+
OPENERS=('class', 'def', 'for', 'if', 'match', 'try',
1632+
'while', 'with')):
16371633
if self.finished:
16381634
pass
16391635
elif type == NAME and token in OPENERS:
@@ -1643,6 +1639,10 @@ def tokeneater(self, type, token, start, end, line,
16431639
self.finished = 1
16441640

16451641
def run(self):
1642+
"""Return 2 lines containing block opener and and indent.
1643+
1644+
Either the indent line or both may be None.
1645+
"""
16461646
try:
16471647
tokens = tokenize.generate_tokens(self.readline)
16481648
for token in tokens:
@@ -1654,6 +1654,7 @@ def run(self):
16541654

16551655
### end autoindent code ###
16561656

1657+
16571658
def prepstr(s):
16581659
"""Extract the underscore from a string.
16591660

‎Lib/idlelib/idle_test/test_editor.py

+33-3
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
"Test editor, coverage 35%."
1+
"Test editor, coverage 53%."
22

33
from idlelib import editor
44
import unittest
55
from collections import namedtuple
66
from test.support import requires
7-
from tkinter import Tk
7+
from tkinter import Tk, Text
88

99
Editor = editor.EditorWindow
1010

@@ -31,7 +31,7 @@ def test_init(self):
3131
e._close()
3232

3333

34-
class TestGetLineIndent(unittest.TestCase):
34+
class GetLineIndentTest(unittest.TestCase):
3535
def test_empty_lines(self):
3636
for tabwidth in [1, 2, 4, 6, 8]:
3737
for line in ['', '\n']:
@@ -181,6 +181,36 @@ def test_indent_and_newline_event(self):
181181
eq(get('1.0', 'end'), ' def f1(self, a,\n \n return a + b\n')
182182

183183

184+
class IndentSearcherTest(unittest.TestCase):
185+
186+
@classmethod
187+
def setUpClass(cls):
188+
requires('gui')
189+
cls.root = Tk()
190+
cls.root.withdraw()
191+
cls.text = Text(cls.root)
192+
193+
@classmethod
194+
def tearDownClass(cls):
195+
cls.root.destroy()
196+
del cls.root
197+
198+
def test_searcher(self):
199+
text = self.text
200+
searcher = (self.text)
201+
test_info = (# text, (block, indent))
202+
("", (None, None)),
203+
("[1,", (None, None)), # TokenError
204+
("if 1:\n", ('if 1:\n', None)),
205+
("if 1:\n 2\n 3\n", ('if 1:\n', ' 2\n')),
206+
)
207+
for code, expected_pair in test_info:
208+
with self.subTest(code=code):
209+
insert(text, code)
210+
actual_pair = editor.IndentSearcher(text).run()
211+
self.assertEqual(actual_pair, expected_pair)
212+
213+
184214
class RMenuTest(unittest.TestCase):
185215

186216
@classmethod

‎Lib/idlelib/idle_test/test_iomenu.py

+6
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,12 @@
88
from idlelib import util
99
from idlelib.idle_test.mock_idle import Func
1010

11+
# Fail if either tokenize.open and t.detect_encoding does not exist.
12+
# These are used in loadfile and encode.
13+
# Also used in pyshell.MI.execfile and runscript.tabnanny.
14+
from tokenize import open, detect_encoding
15+
# Remove when we have proper tests that use both.
16+
1117

1218
class IOBindingTest(unittest.TestCase):
1319

‎Misc/NEWS.d/next/IDLE/2023-05-23-17-19-49.gh-issue-104719.rvYXH-.rst

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Remove IDLE's modification of tokenize.tabsize and test other uses of
2+
tokenize data and methods.

0 commit comments

Comments
 (0)
0