[3.12] gh-111031: Check more files in test_tokenize (GH-111032) (#111061)

miss-islington · sobolevn · web-flow · commit 40db34c90126 · 2023-10-19T09:18:22.000Z
gh-111031: Check more files in `test_tokenize` (GH-111032) (cherry picked from commit e9b5399) Co-authored-by: Nikita Sobolev <mail@sobolevn.me>
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
@@ -1898,19 +1898,9 @@ def test_random_files(self):
         tempdir = os.path.dirname(__file__) or os.curdir
         testfiles = glob.glob(os.path.join(glob.escape(tempdir), "test*.py"))
 
-        # Tokenize is broken on test_pep3131.py because regular expressions are
-        # broken on the obscure unicode identifiers in it. *sigh*
-        # With roundtrip extended to test the 5-tuple mode of untokenize,
-        # 7 more testfiles fail.  Remove them also until the failure is diagnosed.
-
-        testfiles.remove(os.path.join(tempdir, "test_unicode_identifiers.py"))
-
         # TODO: Remove this once we can untokenize PEP 701 syntax
         testfiles.remove(os.path.join(tempdir, "test_fstring.py"))
 
-        for f in ('buffer', 'builtin', 'fileio', 'os', 'platform', 'sys'):
-            testfiles.remove(os.path.join(tempdir, "test_%s.py") % f)
-
         if not support.is_resource_enabled("cpu"):
             testfiles = random.sample(testfiles, 10)
 
