Use echo to improve tokenization

RustPython · MegasKomnenos · Apr 8, 2023 · Apr 10, 2023 · Apr 10, 2023 · Apr 10, 2023
commit 915d53efcb24edac0d70a947ff0cea2be4225b81
diff --git a/scripts/clib/clib_list.txt b/scripts/clib/clib_list.txt
@@ -1,3 +1,156 @@
-__future__ __hello__ _collections_abc _compat_pickle _compression _markupbase _osx_support _py_abc _pycodecs _pydecimal _pyio _sitebuiltins _threading_local _weakrefset abc aifc antigravity argparse ast asynchat asyncore base64 bdb binhex bisect bz2 calendar cgi cgitb chunk cmd code codecs codeop colorsys compileall configparser contextlib contextvars copy copyreg csv dataclasses datetime decimal difflib dis doctest enum filecmp fileinput fnmatch formatter fractions ftplib functools gc genericpath getopt getpass gettext glob graphlib gzip hashlib heapq hmac imghdr imp inspect io ipaddress keyword linecache locale mailbox mimetypes netrc nntplib ntpath nturl2path numbers opcode operator optparse os pathlib pdb pickle pickletools pkgutil platform plistlib posixpath pprint pty py_compile pydoc queue quopri random re reprlib rlcompleter runpy sched secrets selectors shelve shlex shutil site smtpd smtplib sndhdr socket socketserver sre_compile sre_constants sre_parse ssl stat statistics string stringprep struct subprocess sunau sysconfig tabnanny tarfile telnetlib tempfile textwrap this threading timeit token tokenize trace traceback tty types typing uu uuid warnings weakref webbrowser xdrlib zipapp zipfile zipimport
+__future__
+__hello__
+_collections_abc
+_compat_pickle
+_compression
+_markupbase
+_osx_support
+_py_abc
+_pycodecs
+_pydecimal
+_pyio
+_sitebuiltins
+_threading_local
+_weakrefset
+abc
+aifc
+antigravity
+argparse
+ast
+asynchat
+asyncore
+base64
+bdb
+binhex
+bisect
+bz2
+calendar
+cgi
+cgitb
+chunk
+cmd
+code
+codecs
+codeop
+colorsys
+compileall
+configparser
+contextlib
+contextvars
+copy
+copyreg
+csv
+dataclasses
+datetime
+decimal
+difflib
+dis
+doctest
+enum
+filecmp
+fileinput
+fnmatch
+formatter
+fractions
+ftplib
+functools
+gc
+genericpath
+getopt
+getpass
+gettext
+glob
+graphlib
+gzip
+hashlib
+heapq
+hmac
+imghdr
+imp
+inspect
+io
+ipaddress
+keyword
+linecache
+locale
+mailbox
+mimetypes
+netrc
+nntplib
+ntpath
+nturl2path
+numbers
+opcode
+operator
+optparse
+os
+pathlib
+pdb
+pickle
+pickletools
+pkgutil
+platform
+plistlib
+posixpath
+pprint
+pty
+py_compile
+pydoc
+queue
+quopri
+random
+re
+reprlib
+rlcompleter
+runpy
+sched
+secrets
+selectors
+shelve
+shlex
+shutil
+site
+smtpd
+smtplib
+sndhdr
+socket
+socketserver
+sre_compile
+sre_constants
+sre_parse
+ssl
+stat
+statistics
+string
+stringprep
+struct
+subprocess
+sunau
+sysconfig
+tabnanny
+tarfile
+telnetlib
+tempfile
+textwrap
+this
+threading
+timeit
+token
+tokenize
+trace
+traceback
+tty
+types
+typing
+uu
+uuid
+warnings
+weakref
+webbrowser
+xdrlib
+zipapp
+zipfile
+zipimport
+
 # Components listed below are excluded from being tested
 signal
diff --git a/scripts/clib/clib_test_all.sh b/scripts/clib/clib_test_all.sh
@@ -9,7 +9,8 @@ index=1
 
 while :
 do
-    token=$(cat clib_list.txt | tr '\n' ' ' | tr '  ' ' ' | tr '  ' ' ' | cut -d ' ' -f $index)
+    token=$(cat clib_list.txt)
+    token=$(echo $token | cut -d ' ' -f $index)
     if [ -z $token ] || [ $token == "#" ]; then
         break
     fi

diff --git a/scripts/clib/clib_test_one.sh b/scripts/clib/clib_test_one.sh
@@ -13,7 +13,8 @@ index=1
 
 while :
 do
-    token=$(cat clib_list.txt | tr '\n' ' ' | tr '  ' ' ' | tr '  ' ' ' | cut -d ' ' -f $index)
+    token=$(cat clib_list.txt)
+    token=$(echo $token | cut -d ' ' -f $index)
     if [ -z $token ] || [ $token == "#" ]; then
         break
     fi