From 8bd2bf3c16e02cef927f4010fe8b1c4119be57e9 Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Sat, 8 Apr 2023 22:53:29 +0900 Subject: [PATCH 01/17] Add bash script to autotest cpython lib To use, write down your local cpython path in clib_path.txt, and run, bash clib_test_one.sh and check the result in clib_out.txt The script will try to test every component in clib_list.txt, where, clib_test_one.sh : test each component individually clib_test_all.sh : test every components simultaneously --- clib_list.txt | 1 + clib_out.txt | 0 clib_path.txt | 1 + clib_test_all.sh | 66 ++++++++++++++++++++++++++++++++++++++++ clib_test_one.sh | 78 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 146 insertions(+) create mode 100644 clib_list.txt create mode 100644 clib_out.txt create mode 100644 clib_path.txt create mode 100644 clib_test_all.sh create mode 100644 clib_test_one.sh diff --git a/clib_list.txt b/clib_list.txt new file mode 100644 index 0000000000..201217f91f --- /dev/null +++ b/clib_list.txt @@ -0,0 +1 @@ +__future__ __hello__ _collections_abc _compat_pickle _compression _markupbase _osx_support _py_abc _pycodecs _pydecimal _pyio _sitebuiltins _threading_local _weakrefset abc aifc antigravity argparse ast asynchat asyncore base64 bdb binhex bisect bz2 calendar cgi cgitb chunk cmd code codecs codeop colorsys compileall configparser contextlib contextvars copy copyreg csv dataclasses datetime decimal difflib dis doctest enum filecmp fileinput fnmatch formatter fractions ftplib functools gc genericpath getopt getpass gettext glob graphlib gzip hashlib heapq hmac imghdr imp inspect io ipaddress keyword linecache locale mailbox mimetypes netrc nntplib ntpath nturl2path numbers opcode operator optparse os pathlib pdb pickle pickletools pkgutil platform plistlib posixpath pprint pty py_compile pydoc queue quopri random re reprlib rlcompleter runpy sched secrets selectors shelve shlex shutil signal site smtpd smtplib sndhdr socket socketserver sre_compile sre_constants sre_parse ssl stat statistics string stringprep struct subprocess sunau sysconfig tabnanny tarfile telnetlib tempfile textwrap this threading timeit token tokenize trace traceback tty types typing uu uuid warnings weakref webbrowser xdrlib zipapp zipfile zipimport \ No newline at end of file diff --git a/clib_out.txt b/clib_out.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/clib_path.txt b/clib_path.txt new file mode 100644 index 0000000000..fb11140223 --- /dev/null +++ b/clib_path.txt @@ -0,0 +1 @@ +../cpython \ No newline at end of file diff --git a/clib_test_all.sh b/clib_test_all.sh new file mode 100644 index 0000000000..8dc8474942 --- /dev/null +++ b/clib_test_all.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +CPYTHONPATH=$(cat clib_path.txt) + +cp -r "Lib" "LibTmp" + +index=1 + +while : +do + token=$(cat clib_list.txt | cut -d ' ' -f $index) + if [ -z $token ]; then + break + fi + + if [ -f "${CPYTHONPATH}/Lib/${token}.py" ]; then + cp "${CPYTHONPATH}/Lib/${token}.py" "Lib/${token}.py" + fi + + if [ -f "${CPYTHONPATH}/Lib/test/test_${token}.py" ]; then + cp "${CPYTHONPATH}/Lib/test/test_${token}.py" "Lib/test/test_${token}.py" + fi + + ((index++)) +done + +cargo build -r + +if [ -f "clib_out.txt" ]; then + rm "clib_out.txt" +fi + +touch "clib_out.txt" + +index=1 + +while : +do + token=$(cat clib_list.txt | cut -d ' ' -f $index) + if [ -z $token ]; then + break + fi + + message="${token}:" + + if [ -f "Lib/test/test_${token}.py" ]; then + test=$(cargo run -q -r "Lib/test/test_${token}.py" -q 2>&1 >/dev/null | grep "OK") + + if [ ! -z "${test}" ] ; then + message="${message} OK" + fi + if [ -z "${test}" ] ; then + message="${message} FAILED" + fi + fi + if [ ! -f "Lib/test/test_${token}.py" ]; then + message="${message} NOTEST" + fi + + echo $message >> "clib_out.txt" + + ((index++)) +done + +rm -r "Lib" +mv -r "LibTmp" "Lib" \ No newline at end of file diff --git a/clib_test_one.sh b/clib_test_one.sh new file mode 100644 index 0000000000..ee1812b034 --- /dev/null +++ b/clib_test_one.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +CPYTHONPATH=$(cat clib_path.txt) + +if [ -f "clib_out.txt" ]; then + rm "clib_out.txt" +fi + +touch "clib_out.txt" + +index=1 + +while : +do + token=$(cat clib_list.txt | cut -d ' ' -f $index) + if [ -z $token ]; then + break + fi + + lib_exist=false + test_exist=false + test_do=true + message="${token}:" + + if [ -f "Lib/${token}.py" ]; then + lib_exist=true + cp "Lib/${token}.py" "Lib/${token}_tmp_cp.py" + fi + if [ -f "Lib/test/test_${token}.py" ]; then + test_exist=true + cp "Lib/test/test_${token}.py" "Lib/test/test_${token}_tmp_cp.py" + fi + + if [ -f "${CPYTHONPATH}/Lib/${token}.py" ]; then + cp "${CPYTHONPATH}/Lib/${token}.py" "Lib/${token}.py" + fi + if [ ! -f "${CPYTHONPATH}/Lib/${token}.py" ]; then + test_do=false + message="${message} No cpython/Lib/${token}.py" + fi + + if [ -f "${CPYTHONPATH}/Lib/test/test_${token}.py" ]; then + cp "${CPYTHONPATH}/Lib/test/test_${token}.py" "Lib/test/test_${token}.py" + fi + if [ ! -f "${CPYTHONPATH}/Lib/test/test_${token}.py" ]; then + test_do=false + message="${message} No cpython/Lib/test/test_${token}.py" + fi + + if $test_do ; then + test=$(cargo run -q "Lib/test/test_${token}.py" -q 2>&1 >/dev/null | grep "OK") + + if [ ! -z "${test}" ] ; then + message="${message} OK" + fi + if [ -z "${test}" ] ; then + message="${message} FAILED" + fi + fi + + if $lib_exist ; then + mv "Lib/${token}_tmp_cp.py" "Lib/${token}.py" + fi + if ! $lib_exist && [ -f "Lib/${token}.py" ] ; then + rm "Lib/${token}.py" + fi + + if $test_exist ; then + mv "Lib/test/test_${token}_tmp_cp.py" "Lib/test/test_${token}.py" + fi + if ! $test_exist && [ -f "Lib/test/test_${token}.py" ] ; then + rm "Lib/test/test_${token}.py" + fi + + echo $message >> "clib_out.txt" + + ((index++)) +done \ No newline at end of file From 8ae27c7d75fd0d8cf4d6a0dbc5abfec97626c10b Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Mon, 10 Apr 2023 19:13:40 +0900 Subject: [PATCH 02/17] Add comment functionality to script It works by making the code to ignore every tokens that come after the comment symbol --- clib_test_all.sh | 8 ++++---- clib_test_one.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/clib_test_all.sh b/clib_test_all.sh index 8dc8474942..9df0e91114 100644 --- a/clib_test_all.sh +++ b/clib_test_all.sh @@ -8,8 +8,8 @@ index=1 while : do - token=$(cat clib_list.txt | cut -d ' ' -f $index) - if [ -z $token ]; then + token=$(cat clib_list.txt | tr '\n' ' ' | tr ' ' ' ' | tr ' ' ' ' | cut -d ' ' -f $index) + if [ -z $token ] || [ $token == "#" ]; then break fi @@ -36,8 +36,8 @@ index=1 while : do - token=$(cat clib_list.txt | cut -d ' ' -f $index) - if [ -z $token ]; then + token=$(cat clib_list.txt | tr '\n' ' ' | tr ' ' ' ' | tr ' ' ' ' | cut -d ' ' -f $index) + if [ -z $token ] || [ $token == "#" ]; then break fi diff --git a/clib_test_one.sh b/clib_test_one.sh index ee1812b034..07de16054d 100644 --- a/clib_test_one.sh +++ b/clib_test_one.sh @@ -12,8 +12,8 @@ index=1 while : do - token=$(cat clib_list.txt | cut -d ' ' -f $index) - if [ -z $token ]; then + token=$(cat clib_list.txt | tr '\n' ' ' | tr ' ' ' ' | tr ' ' ' ' | cut -d ' ' -f $index) + if [ -z $token ] || [ $token == "#" ]; then break fi From 540d04a03b18ae0e737851d6b0ef74df1bb206d3 Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Mon, 10 Apr 2023 19:14:58 +0900 Subject: [PATCH 03/17] Comment out signal.py It fails in a weird way and crashes the whole system by gobbling up memory --- clib_list.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clib_list.txt b/clib_list.txt index 201217f91f..7ada67d24e 100644 --- a/clib_list.txt +++ b/clib_list.txt @@ -1 +1,3 @@ -__future__ __hello__ _collections_abc _compat_pickle _compression _markupbase _osx_support _py_abc _pycodecs _pydecimal _pyio _sitebuiltins _threading_local _weakrefset abc aifc antigravity argparse ast asynchat asyncore base64 bdb binhex bisect bz2 calendar cgi cgitb chunk cmd code codecs codeop colorsys compileall configparser contextlib contextvars copy copyreg csv dataclasses datetime decimal difflib dis doctest enum filecmp fileinput fnmatch formatter fractions ftplib functools gc genericpath getopt getpass gettext glob graphlib gzip hashlib heapq hmac imghdr imp inspect io ipaddress keyword linecache locale mailbox mimetypes netrc nntplib ntpath nturl2path numbers opcode operator optparse os pathlib pdb pickle pickletools pkgutil platform plistlib posixpath pprint pty py_compile pydoc queue quopri random re reprlib rlcompleter runpy sched secrets selectors shelve shlex shutil signal site smtpd smtplib sndhdr socket socketserver sre_compile sre_constants sre_parse ssl stat statistics string stringprep struct subprocess sunau sysconfig tabnanny tarfile telnetlib tempfile textwrap this threading timeit token tokenize trace traceback tty types typing uu uuid warnings weakref webbrowser xdrlib zipapp zipfile zipimport \ No newline at end of file +__future__ __hello__ _collections_abc _compat_pickle _compression _markupbase _osx_support _py_abc _pycodecs _pydecimal _pyio _sitebuiltins _threading_local _weakrefset abc aifc antigravity argparse ast asynchat asyncore base64 bdb binhex bisect bz2 calendar cgi cgitb chunk cmd code codecs codeop colorsys compileall configparser contextlib contextvars copy copyreg csv dataclasses datetime decimal difflib dis doctest enum filecmp fileinput fnmatch formatter fractions ftplib functools gc genericpath getopt getpass gettext glob graphlib gzip hashlib heapq hmac imghdr imp inspect io ipaddress keyword linecache locale mailbox mimetypes netrc nntplib ntpath nturl2path numbers opcode operator optparse os pathlib pdb pickle pickletools pkgutil platform plistlib posixpath pprint pty py_compile pydoc queue quopri random re reprlib rlcompleter runpy sched secrets selectors shelve shlex shutil site smtpd smtplib sndhdr socket socketserver sre_compile sre_constants sre_parse ssl stat statistics string stringprep struct subprocess sunau sysconfig tabnanny tarfile telnetlib tempfile textwrap this threading timeit token tokenize trace traceback tty types typing uu uuid warnings weakref webbrowser xdrlib zipapp zipfile zipimport +# Components listed below are excluded from being tested +signal \ No newline at end of file From 0ac973d9bdac0980f96e4136ee09c2a9c7817a41 Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Mon, 10 Apr 2023 19:20:44 +0900 Subject: [PATCH 04/17] Move clib scripts to scripts/clib --- clib_path.txt | 1 - clib_list.txt => scripts/clib/clib_list.txt | 0 clib_out.txt => scripts/clib/clib_out.txt | 0 scripts/clib/clib_path.txt | 1 + .../clib/clib_test_all.sh | 17 ++++++------ .../clib/clib_test_one.sh | 27 ++++++++++--------- 6 files changed, 24 insertions(+), 22 deletions(-) delete mode 100644 clib_path.txt rename clib_list.txt => scripts/clib/clib_list.txt (100%) rename clib_out.txt => scripts/clib/clib_out.txt (100%) create mode 100644 scripts/clib/clib_path.txt rename clib_test_all.sh => scripts/clib/clib_test_all.sh (63%) rename clib_test_one.sh => scripts/clib/clib_test_one.sh (54%) diff --git a/clib_path.txt b/clib_path.txt deleted file mode 100644 index fb11140223..0000000000 --- a/clib_path.txt +++ /dev/null @@ -1 +0,0 @@ -../cpython \ No newline at end of file diff --git a/clib_list.txt b/scripts/clib/clib_list.txt similarity index 100% rename from clib_list.txt rename to scripts/clib/clib_list.txt diff --git a/clib_out.txt b/scripts/clib/clib_out.txt similarity index 100% rename from clib_out.txt rename to scripts/clib/clib_out.txt diff --git a/scripts/clib/clib_path.txt b/scripts/clib/clib_path.txt new file mode 100644 index 0000000000..ceda6b6ece --- /dev/null +++ b/scripts/clib/clib_path.txt @@ -0,0 +1 @@ +../../../cpython \ No newline at end of file diff --git a/clib_test_all.sh b/scripts/clib/clib_test_all.sh similarity index 63% rename from clib_test_all.sh rename to scripts/clib/clib_test_all.sh index 9df0e91114..e20381ca74 100644 --- a/clib_test_all.sh +++ b/scripts/clib/clib_test_all.sh @@ -1,8 +1,9 @@ #!/bin/bash CPYTHONPATH=$(cat clib_path.txt) +RPYTHONPATH="../.." -cp -r "Lib" "LibTmp" +cp -r "${RPYTHONPATH}/Lib" "${RPYTHONPATH}/LibTmp" index=1 @@ -14,11 +15,11 @@ do fi if [ -f "${CPYTHONPATH}/Lib/${token}.py" ]; then - cp "${CPYTHONPATH}/Lib/${token}.py" "Lib/${token}.py" + cp "${CPYTHONPATH}/Lib/${token}.py" "${RPYTHONPATH}/Lib/${token}.py" fi if [ -f "${CPYTHONPATH}/Lib/test/test_${token}.py" ]; then - cp "${CPYTHONPATH}/Lib/test/test_${token}.py" "Lib/test/test_${token}.py" + cp "${CPYTHONPATH}/Lib/test/test_${token}.py" "${RPYTHONPATH}/Lib/test/test_${token}.py" fi ((index++)) @@ -43,8 +44,8 @@ do message="${token}:" - if [ -f "Lib/test/test_${token}.py" ]; then - test=$(cargo run -q -r "Lib/test/test_${token}.py" -q 2>&1 >/dev/null | grep "OK") + if [ -f "${RPYTHONPATH}/Lib/test/test_${token}.py" ]; then + test=$(cargo run -q -r "${RPYTHONPATH}/Lib/test/test_${token}.py" -q 2>&1 >/dev/null | grep "OK") if [ ! -z "${test}" ] ; then message="${message} OK" @@ -53,7 +54,7 @@ do message="${message} FAILED" fi fi - if [ ! -f "Lib/test/test_${token}.py" ]; then + if [ ! -f "${RPYTHONPATH}/Lib/test/test_${token}.py" ]; then message="${message} NOTEST" fi @@ -62,5 +63,5 @@ do ((index++)) done -rm -r "Lib" -mv -r "LibTmp" "Lib" \ No newline at end of file +rm -r "${RPYTHONPATH}/Lib" +mv -r "${RPYTHONPATH}/LibTmp" "${RPYTHONPATH}/Lib" \ No newline at end of file diff --git a/clib_test_one.sh b/scripts/clib/clib_test_one.sh similarity index 54% rename from clib_test_one.sh rename to scripts/clib/clib_test_one.sh index 07de16054d..cf74e6aa9b 100644 --- a/clib_test_one.sh +++ b/scripts/clib/clib_test_one.sh @@ -1,6 +1,7 @@ #!/bin/bash CPYTHONPATH=$(cat clib_path.txt) +RPYTHONPATH="../.." if [ -f "clib_out.txt" ]; then rm "clib_out.txt" @@ -22,17 +23,17 @@ do test_do=true message="${token}:" - if [ -f "Lib/${token}.py" ]; then + if [ -f "${RPYTHONPATH}/Lib/${token}.py" ]; then lib_exist=true - cp "Lib/${token}.py" "Lib/${token}_tmp_cp.py" + cp "${RPYTHONPATH}/Lib/${token}.py" "${RPYTHONPATH}/Lib/${token}_tmp_cp.py" fi - if [ -f "Lib/test/test_${token}.py" ]; then + if [ -f "${RPYTHONPATH}/Lib/test/test_${token}.py" ]; then test_exist=true - cp "Lib/test/test_${token}.py" "Lib/test/test_${token}_tmp_cp.py" + cp "${RPYTHONPATH}/Lib/test/test_${token}.py" "${RPYTHONPATH}/Lib/test/test_${token}_tmp_cp.py" fi if [ -f "${CPYTHONPATH}/Lib/${token}.py" ]; then - cp "${CPYTHONPATH}/Lib/${token}.py" "Lib/${token}.py" + cp "${CPYTHONPATH}/Lib/${token}.py" "${RPYTHONPATH}/Lib/${token}.py" fi if [ ! -f "${CPYTHONPATH}/Lib/${token}.py" ]; then test_do=false @@ -40,7 +41,7 @@ do fi if [ -f "${CPYTHONPATH}/Lib/test/test_${token}.py" ]; then - cp "${CPYTHONPATH}/Lib/test/test_${token}.py" "Lib/test/test_${token}.py" + cp "${CPYTHONPATH}/Lib/test/test_${token}.py" "${RPYTHONPATH}/Lib/test/test_${token}.py" fi if [ ! -f "${CPYTHONPATH}/Lib/test/test_${token}.py" ]; then test_do=false @@ -48,7 +49,7 @@ do fi if $test_do ; then - test=$(cargo run -q "Lib/test/test_${token}.py" -q 2>&1 >/dev/null | grep "OK") + test=$(cargo run -q "${RPYTHONPATH}/Lib/test/test_${token}.py" -q 2>&1 >/dev/null | grep "OK") if [ ! -z "${test}" ] ; then message="${message} OK" @@ -59,17 +60,17 @@ do fi if $lib_exist ; then - mv "Lib/${token}_tmp_cp.py" "Lib/${token}.py" + mv "${RPYTHONPATH}/Lib/${token}_tmp_cp.py" "${RPYTHONPATH}/Lib/${token}.py" fi - if ! $lib_exist && [ -f "Lib/${token}.py" ] ; then - rm "Lib/${token}.py" + if ! $lib_exist && [ -f "${RPYTHONPATH}/Lib/${token}.py" ] ; then + rm "${RPYTHONPATH}/Lib/${token}.py" fi if $test_exist ; then - mv "Lib/test/test_${token}_tmp_cp.py" "Lib/test/test_${token}.py" + mv "${RPYTHONPATH}/Lib/test/test_${token}_tmp_cp.py" "${RPYTHONPATH}/Lib/test/test_${token}.py" fi - if ! $test_exist && [ -f "Lib/test/test_${token}.py" ] ; then - rm "Lib/test/test_${token}.py" + if ! $test_exist && [ -f "${RPYTHONPATH}/Lib/test/test_${token}.py" ] ; then + rm "${RPYTHONPATH}/Lib/test/test_${token}.py" fi echo $message >> "clib_out.txt" From 915d53efcb24edac0d70a947ff0cea2be4225b81 Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Mon, 10 Apr 2023 20:41:48 +0900 Subject: [PATCH 05/17] Use echo to improve tokenization --- scripts/clib/clib_list.txt | 155 +++++++++++++++++++++++++++++++++- scripts/clib/clib_test_all.sh | 3 +- scripts/clib/clib_test_one.sh | 3 +- 3 files changed, 158 insertions(+), 3 deletions(-) diff --git a/scripts/clib/clib_list.txt b/scripts/clib/clib_list.txt index 7ada67d24e..4254a0896a 100644 --- a/scripts/clib/clib_list.txt +++ b/scripts/clib/clib_list.txt @@ -1,3 +1,156 @@ -__future__ __hello__ _collections_abc _compat_pickle _compression _markupbase _osx_support _py_abc _pycodecs _pydecimal _pyio _sitebuiltins _threading_local _weakrefset abc aifc antigravity argparse ast asynchat asyncore base64 bdb binhex bisect bz2 calendar cgi cgitb chunk cmd code codecs codeop colorsys compileall configparser contextlib contextvars copy copyreg csv dataclasses datetime decimal difflib dis doctest enum filecmp fileinput fnmatch formatter fractions ftplib functools gc genericpath getopt getpass gettext glob graphlib gzip hashlib heapq hmac imghdr imp inspect io ipaddress keyword linecache locale mailbox mimetypes netrc nntplib ntpath nturl2path numbers opcode operator optparse os pathlib pdb pickle pickletools pkgutil platform plistlib posixpath pprint pty py_compile pydoc queue quopri random re reprlib rlcompleter runpy sched secrets selectors shelve shlex shutil site smtpd smtplib sndhdr socket socketserver sre_compile sre_constants sre_parse ssl stat statistics string stringprep struct subprocess sunau sysconfig tabnanny tarfile telnetlib tempfile textwrap this threading timeit token tokenize trace traceback tty types typing uu uuid warnings weakref webbrowser xdrlib zipapp zipfile zipimport +__future__ +__hello__ +_collections_abc +_compat_pickle +_compression +_markupbase +_osx_support +_py_abc +_pycodecs +_pydecimal +_pyio +_sitebuiltins +_threading_local +_weakrefset +abc +aifc +antigravity +argparse +ast +asynchat +asyncore +base64 +bdb +binhex +bisect +bz2 +calendar +cgi +cgitb +chunk +cmd +code +codecs +codeop +colorsys +compileall +configparser +contextlib +contextvars +copy +copyreg +csv +dataclasses +datetime +decimal +difflib +dis +doctest +enum +filecmp +fileinput +fnmatch +formatter +fractions +ftplib +functools +gc +genericpath +getopt +getpass +gettext +glob +graphlib +gzip +hashlib +heapq +hmac +imghdr +imp +inspect +io +ipaddress +keyword +linecache +locale +mailbox +mimetypes +netrc +nntplib +ntpath +nturl2path +numbers +opcode +operator +optparse +os +pathlib +pdb +pickle +pickletools +pkgutil +platform +plistlib +posixpath +pprint +pty +py_compile +pydoc +queue +quopri +random +re +reprlib +rlcompleter +runpy +sched +secrets +selectors +shelve +shlex +shutil +site +smtpd +smtplib +sndhdr +socket +socketserver +sre_compile +sre_constants +sre_parse +ssl +stat +statistics +string +stringprep +struct +subprocess +sunau +sysconfig +tabnanny +tarfile +telnetlib +tempfile +textwrap +this +threading +timeit +token +tokenize +trace +traceback +tty +types +typing +uu +uuid +warnings +weakref +webbrowser +xdrlib +zipapp +zipfile +zipimport + # Components listed below are excluded from being tested signal \ No newline at end of file diff --git a/scripts/clib/clib_test_all.sh b/scripts/clib/clib_test_all.sh index e20381ca74..8246a8e949 100644 --- a/scripts/clib/clib_test_all.sh +++ b/scripts/clib/clib_test_all.sh @@ -9,7 +9,8 @@ index=1 while : do - token=$(cat clib_list.txt | tr '\n' ' ' | tr ' ' ' ' | tr ' ' ' ' | cut -d ' ' -f $index) + token=$(cat clib_list.txt) + token=$(echo $token | cut -d ' ' -f $index) if [ -z $token ] || [ $token == "#" ]; then break fi diff --git a/scripts/clib/clib_test_one.sh b/scripts/clib/clib_test_one.sh index cf74e6aa9b..e655f15b54 100644 --- a/scripts/clib/clib_test_one.sh +++ b/scripts/clib/clib_test_one.sh @@ -13,7 +13,8 @@ index=1 while : do - token=$(cat clib_list.txt | tr '\n' ' ' | tr ' ' ' ' | tr ' ' ' ' | cut -d ' ' -f $index) + token=$(cat clib_list.txt) + token=$(echo $token | cut -d ' ' -f $index) if [ -z $token ] || [ $token == "#" ]; then break fi From b3eca666c3dab75194f05dec784b3d42e8bc298d Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Mon, 10 Apr 2023 20:42:05 +0900 Subject: [PATCH 06/17] Script output --- scripts/clib/clib_out.txt | 153 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/scripts/clib/clib_out.txt b/scripts/clib/clib_out.txt index e69de29bb2..7be1e37c9c 100644 --- a/scripts/clib/clib_out.txt +++ b/scripts/clib/clib_out.txt @@ -0,0 +1,153 @@ +__future__: OK +__hello__: No cpython/Lib/test/test___hello__.py +_collections_abc: No cpython/Lib/test/test__collections_abc.py +_compat_pickle: No cpython/Lib/test/test__compat_pickle.py +_compression: No cpython/Lib/test/test__compression.py +_markupbase: No cpython/Lib/test/test__markupbase.py +_osx_support: OK +_py_abc: No cpython/Lib/test/test__py_abc.py +_pycodecs: No cpython/Lib/_pycodecs.py No cpython/Lib/test/test__pycodecs.py +_pydecimal: No cpython/Lib/test/test__pydecimal.py +_pyio: No cpython/Lib/test/test__pyio.py +_sitebuiltins: No cpython/Lib/test/test__sitebuiltins.py +_threading_local: No cpython/Lib/test/test__threading_local.py +_weakrefset: No cpython/Lib/test/test__weakrefset.py +abc: FAILED +aifc: FAILED +antigravity: No cpython/Lib/test/test_antigravity.py +argparse: OK +ast: FAILED +asynchat: FAILED +asyncore: FAILED +base64: OK +bdb: FAILED +binhex: No cpython/Lib/binhex.py No cpython/Lib/test/test_binhex.py +bisect: OK +bz2: FAILED +calendar: OK +cgi: FAILED +cgitb: FAILED +chunk: No cpython/Lib/test/test_chunk.py +cmd: OK +code: FAILED +codecs: FAILED +codeop: FAILED +colorsys: OK +compileall: FAILED +configparser: OK +contextlib: FAILED +contextvars: No cpython/Lib/test/test_contextvars.py +copy: FAILED +copyreg: OK +csv: FAILED +dataclasses: FAILED +datetime: FAILED +decimal: FAILED +difflib: FAILED +dis: FAILED +doctest: FAILED +enum: FAILED +filecmp: OK +fileinput: FAILED +fnmatch: FAILED +formatter: No cpython/Lib/formatter.py No cpython/Lib/test/test_formatter.py +fractions: FAILED +ftplib: OK +functools: FAILED +gc: No cpython/Lib/gc.py +genericpath: OK +getopt: OK +getpass: OK +gettext: OK +glob: OK +graphlib: OK +gzip: OK +hashlib: FAILED +heapq: OK +hmac: OK +imghdr: FAILED +imp: FAILED +inspect: FAILED +io: FAILED +ipaddress: OK +keyword: OK +linecache: FAILED +locale: OK +mailbox: FAILED +mimetypes: OK +netrc: FAILED +nntplib: FAILED +ntpath: FAILED +nturl2path: No cpython/Lib/test/test_nturl2path.py +numbers: No cpython/Lib/test/test_numbers.py +opcode: No cpython/Lib/test/test_opcode.py +operator: OK +optparse: OK +os: FAILED +pathlib: FAILED +pdb: FAILED +pickle: FAILED +pickletools: FAILED +pkgutil: FAILED +platform: OK +plistlib: FAILED +posixpath: FAILED +pprint: FAILED +pty: FAILED +py_compile: FAILED +pydoc: FAILED +queue: OK +quopri: OK +random: FAILED +re: No cpython/Lib/re.py +reprlib: FAILED +rlcompleter: FAILED +runpy: FAILED +sched: OK +secrets: OK +selectors: OK +shelve: FAILED +shlex: FAILED +shutil: FAILED +site: FAILED +smtpd: FAILED +smtplib: FAILED +sndhdr: FAILED +socket: FAILED +socketserver: OK +sre_compile: No cpython/Lib/test/test_sre_compile.py +sre_constants: No cpython/Lib/test/test_sre_constants.py +sre_parse: No cpython/Lib/test/test_sre_parse.py +ssl: FAILED +stat: FAILED +statistics: FAILED +string: OK +stringprep: FAILED +struct: FAILED +subprocess: FAILED +sunau: FAILED +sysconfig: FAILED +tabnanny: FAILED +tarfile: FAILED +telnetlib: FAILED +tempfile: FAILED +textwrap: OK +this: No cpython/Lib/test/test_this.py +threading: FAILED +timeit: FAILED +token: No cpython/Lib/test/test_token.py +tokenize: FAILED +trace: FAILED +traceback: FAILED +tty: No cpython/Lib/test/test_tty.py +types: FAILED +typing: FAILED +uu: FAILED +uuid: FAILED +warnings: No cpython/Lib/test/test_warnings.py +weakref: FAILED +webbrowser: OK +xdrlib: FAILED +zipapp: FAILED +zipfile: FAILED +zipimport: OK From 07515e3a98d8e9d57d695ebdd3788c9d01d137b8 Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Mon, 10 Apr 2023 20:44:25 +0900 Subject: [PATCH 07/17] Add readme --- scripts/clib/readme.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 scripts/clib/readme.txt diff --git a/scripts/clib/readme.txt b/scripts/clib/readme.txt new file mode 100644 index 0000000000..c71534b810 --- /dev/null +++ b/scripts/clib/readme.txt @@ -0,0 +1,11 @@ +To use, write down your local cpython path in clib_path.txt, and run, +bash clib_test_one.sh + +and check the result in clib_out.txt + +The script will try to test every component in clib_list.txt, where, + +clib_test_one.sh : test each component individually +clib_test_all.sh : test every components simultaneously + +Use of clib_test_all.sh is discouraged, as one failing component can lead to cascading failure of otherwise non-failing components \ No newline at end of file From 95de4b10f0e03a59da78b6d6c1355e47235fdb3a Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Tue, 11 Apr 2023 09:48:11 +0900 Subject: [PATCH 08/17] Apply feedback on readme --- scripts/clib/{readme.txt => readme.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename scripts/clib/{readme.txt => readme.md} (87%) diff --git a/scripts/clib/readme.txt b/scripts/clib/readme.md similarity index 87% rename from scripts/clib/readme.txt rename to scripts/clib/readme.md index c71534b810..b587ed02d1 100644 --- a/scripts/clib/readme.txt +++ b/scripts/clib/readme.md @@ -6,6 +6,6 @@ and check the result in clib_out.txt The script will try to test every component in clib_list.txt, where, clib_test_one.sh : test each component individually -clib_test_all.sh : test every components simultaneously +clib_test_all.sh : test every component simultaneously Use of clib_test_all.sh is discouraged, as one failing component can lead to cascading failure of otherwise non-failing components \ No newline at end of file From f20a9d00841cf08bb2a531c4888133932e32e5ea Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Tue, 11 Apr 2023 10:02:15 +0900 Subject: [PATCH 09/17] Change how comment works in clib_list.txt To use, add # in front of any line that you want to comment out Warning, it won't work unless # is the first character in a string, not including whitespaces --- scripts/clib/clib_list.txt | 5 +++-- scripts/clib/clib_test_all.sh | 17 ++++++++++++----- scripts/clib/clib_test_one.sh | 9 ++++++--- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/scripts/clib/clib_list.txt b/scripts/clib/clib_list.txt index 4254a0896a..b4a433bd8e 100644 --- a/scripts/clib/clib_list.txt +++ b/scripts/clib/clib_list.txt @@ -109,6 +109,7 @@ selectors shelve shlex shutil +#signal site smtpd smtplib @@ -152,5 +153,5 @@ zipapp zipfile zipimport -# Components listed below are excluded from being tested -signal \ No newline at end of file +# EOF marks the end of file +EOF \ No newline at end of file diff --git a/scripts/clib/clib_test_all.sh b/scripts/clib/clib_test_all.sh index 8246a8e949..d73e486c89 100644 --- a/scripts/clib/clib_test_all.sh +++ b/scripts/clib/clib_test_all.sh @@ -9,9 +9,12 @@ index=1 while : do - token=$(cat clib_list.txt) - token=$(echo $token | cut -d ' ' -f $index) - if [ -z $token ] || [ $token == "#" ]; then + token=$(sed -n ${index}p clib_list.txt | xargs) + if [ -z $token ] || [ ${token::1} == "#" ]; then + ((index++)) + continue + fi + if [ $token == "EOF" ]; then break fi @@ -38,8 +41,12 @@ index=1 while : do - token=$(cat clib_list.txt | tr '\n' ' ' | tr ' ' ' ' | tr ' ' ' ' | cut -d ' ' -f $index) - if [ -z $token ] || [ $token == "#" ]; then + token=$(sed -n ${index}p clib_list.txt | xargs) + if [ -z $token ] || [ ${token::1} == "#" ]; then + ((index++)) + continue + fi + if [ $token == "EOF" ]; then break fi diff --git a/scripts/clib/clib_test_one.sh b/scripts/clib/clib_test_one.sh index e655f15b54..cb139be522 100644 --- a/scripts/clib/clib_test_one.sh +++ b/scripts/clib/clib_test_one.sh @@ -13,9 +13,12 @@ index=1 while : do - token=$(cat clib_list.txt) - token=$(echo $token | cut -d ' ' -f $index) - if [ -z $token ] || [ $token == "#" ]; then + token=$(sed -n ${index}p clib_list.txt | xargs) + if [ -z $token ] || [ ${token::1} == "#" ]; then + ((index++)) + continue + fi + if [ $token == "EOF" ]; then break fi From 1c7bda8d705f1fb2b4839b4010e7047f765e5378 Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Tue, 25 Apr 2023 10:55:46 +0900 Subject: [PATCH 10/17] Change script to be in python --- scripts/clib/clib_list.txt | 3 +- scripts/clib/clib_out.txt | 241 +++++++++++++--------------------- scripts/clib/clib_path.txt | 1 - scripts/clib/clib_test.py | 79 +++++++++++ scripts/clib/clib_test_all.sh | 75 ----------- scripts/clib/clib_test_one.sh | 83 ------------ scripts/clib/readme.md | 12 +- 7 files changed, 173 insertions(+), 321 deletions(-) delete mode 100644 scripts/clib/clib_path.txt create mode 100644 scripts/clib/clib_test.py delete mode 100644 scripts/clib/clib_test_all.sh delete mode 100644 scripts/clib/clib_test_one.sh diff --git a/scripts/clib/clib_list.txt b/scripts/clib/clib_list.txt index b4a433bd8e..28ca9e585a 100644 --- a/scripts/clib/clib_list.txt +++ b/scripts/clib/clib_list.txt @@ -153,5 +153,4 @@ zipapp zipfile zipimport -# EOF marks the end of file -EOF \ No newline at end of file +# Make sure file ends with empty space diff --git a/scripts/clib/clib_out.txt b/scripts/clib/clib_out.txt index 7be1e37c9c..77412d6d0e 100644 --- a/scripts/clib/clib_out.txt +++ b/scripts/clib/clib_out.txt @@ -1,153 +1,88 @@ -__future__: OK -__hello__: No cpython/Lib/test/test___hello__.py -_collections_abc: No cpython/Lib/test/test__collections_abc.py -_compat_pickle: No cpython/Lib/test/test__compat_pickle.py -_compression: No cpython/Lib/test/test__compression.py -_markupbase: No cpython/Lib/test/test__markupbase.py -_osx_support: OK -_py_abc: No cpython/Lib/test/test__py_abc.py -_pycodecs: No cpython/Lib/_pycodecs.py No cpython/Lib/test/test__pycodecs.py -_pydecimal: No cpython/Lib/test/test__pydecimal.py -_pyio: No cpython/Lib/test/test__pyio.py -_sitebuiltins: No cpython/Lib/test/test__sitebuiltins.py -_threading_local: No cpython/Lib/test/test__threading_local.py -_weakrefset: No cpython/Lib/test/test__weakrefset.py -abc: FAILED -aifc: FAILED -antigravity: No cpython/Lib/test/test_antigravity.py -argparse: OK -ast: FAILED -asynchat: FAILED -asyncore: FAILED -base64: OK -bdb: FAILED -binhex: No cpython/Lib/binhex.py No cpython/Lib/test/test_binhex.py -bisect: OK -bz2: FAILED -calendar: OK -cgi: FAILED -cgitb: FAILED -chunk: No cpython/Lib/test/test_chunk.py -cmd: OK -code: FAILED -codecs: FAILED -codeop: FAILED -colorsys: OK -compileall: FAILED -configparser: OK -contextlib: FAILED -contextvars: No cpython/Lib/test/test_contextvars.py -copy: FAILED -copyreg: OK -csv: FAILED -dataclasses: FAILED -datetime: FAILED -decimal: FAILED -difflib: FAILED -dis: FAILED -doctest: FAILED -enum: FAILED -filecmp: OK -fileinput: FAILED -fnmatch: FAILED -formatter: No cpython/Lib/formatter.py No cpython/Lib/test/test_formatter.py -fractions: FAILED -ftplib: OK -functools: FAILED -gc: No cpython/Lib/gc.py -genericpath: OK -getopt: OK -getpass: OK -gettext: OK -glob: OK -graphlib: OK -gzip: OK -hashlib: FAILED -heapq: OK -hmac: OK -imghdr: FAILED -imp: FAILED -inspect: FAILED -io: FAILED -ipaddress: OK -keyword: OK -linecache: FAILED -locale: OK -mailbox: FAILED -mimetypes: OK -netrc: FAILED -nntplib: FAILED -ntpath: FAILED -nturl2path: No cpython/Lib/test/test_nturl2path.py -numbers: No cpython/Lib/test/test_numbers.py -opcode: No cpython/Lib/test/test_opcode.py -operator: OK -optparse: OK -os: FAILED -pathlib: FAILED -pdb: FAILED -pickle: FAILED -pickletools: FAILED -pkgutil: FAILED -platform: OK -plistlib: FAILED -posixpath: FAILED -pprint: FAILED -pty: FAILED -py_compile: FAILED -pydoc: FAILED -queue: OK -quopri: OK -random: FAILED -re: No cpython/Lib/re.py -reprlib: FAILED -rlcompleter: FAILED -runpy: FAILED -sched: OK -secrets: OK -selectors: OK -shelve: FAILED -shlex: FAILED -shutil: FAILED -site: FAILED -smtpd: FAILED -smtplib: FAILED -sndhdr: FAILED -socket: FAILED -socketserver: OK -sre_compile: No cpython/Lib/test/test_sre_compile.py -sre_constants: No cpython/Lib/test/test_sre_constants.py -sre_parse: No cpython/Lib/test/test_sre_parse.py -ssl: FAILED -stat: FAILED -statistics: FAILED -string: OK -stringprep: FAILED -struct: FAILED -subprocess: FAILED -sunau: FAILED -sysconfig: FAILED -tabnanny: FAILED -tarfile: FAILED -telnetlib: FAILED -tempfile: FAILED -textwrap: OK -this: No cpython/Lib/test/test_this.py -threading: FAILED -timeit: FAILED -token: No cpython/Lib/test/test_token.py -tokenize: FAILED -trace: FAILED -traceback: FAILED -tty: No cpython/Lib/test/test_tty.py -types: FAILED -typing: FAILED -uu: FAILED -uuid: FAILED -warnings: No cpython/Lib/test/test_warnings.py -weakref: FAILED -webbrowser: OK -xdrlib: FAILED -zipapp: FAILED -zipfile: FAILED -zipimport: OK +__future__: Failed +__hello__: No cpython/Lib/test/test___hello__.py +_collections_abc: No cpython/Lib/test/test__collections_abc.py +_compat_pickle: No cpython/Lib/test/test__compat_pickle.py +_compression: No cpython/Lib/test/test__compression.py +_markupbase: No cpython/Lib/test/test__markupbase.py +_osx_support: Failed +_py_abc: No cpython/Lib/test/test__py_abc.py +_pycodecs: No cpython/Lib/_pycodecs.py No cpython/Lib/test/test__pycodecs.py +_pydecimal: No cpython/Lib/test/test__pydecimal.py +_pyio: No cpython/Lib/test/test__pyio.py +_sitebuiltins: No cpython/Lib/test/test__sitebuiltins.py +_threading_local: No cpython/Lib/test/test__threading_local.py +_weakrefset: No cpython/Lib/test/test__weakrefset.py +abc: Failed +aifc: Failed +antigravity: No cpython/Lib/test/test_antigravity.py +argparse: Failed +ast: Failed +asynchat: Failed +asyncore: Failed +base64: Failed +bdb: Failed +binhex: No cpython/Lib/binhex.py No cpython/Lib/test/test_binhex.py +bisect: Failed +bz2: Failed +calendar: Failed +cgi: Failed +cgitb: Failed +chunk: No cpython/Lib/test/test_chunk.py +cmd: Failed +code: Failed +codecs: Failed +codeop: Failed +colorsys: Failed +compileall: Failed +configparser: Failed +contextlib: Failed +contextvars: No cpython/Lib/test/test_contextvars.py +copy: Failed +copyreg: Failed +csv: Failed +dataclasses: Failed +datetime: Failed +decimal: Failed +difflib: Failed +dis: Failed +doctest: Failed +enum: Failed +filecmp: Failed +fileinput: Failed +fnmatch: Failed +formatter: No cpython/Lib/formatter.py No cpython/Lib/test/test_formatter.py +fractions: Failed +ftplib: Failed +functools: Failed +gc: No cpython/Lib/gc.py +genericpath: Failed +getopt: Failed +getpass: Failed +gettext: Failed +glob: Failed +graphlib: Failed +gzip: Failed +hashlib: Failed +heapq: Failed +hmac: Failed +imghdr: Failed +imp: Failed +inspect: Failed +io: Failed +ipaddress: Failed +keyword: Failed +linecache: Failed +locale: Failed +mailbox: Failed +mimetypes: Failed +netrc: OK +nntplib: Failed +ntpath: Failed +nturl2path: No cpython/Lib/test/test_nturl2path.py +numbers: No cpython/Lib/test/test_numbers.py +opcode: No cpython/Lib/test/test_opcode.py +operator: Failed +optparse: Failed +os: Failed +pathlib: Failed +pdb: Failed diff --git a/scripts/clib/clib_path.txt b/scripts/clib/clib_path.txt deleted file mode 100644 index ceda6b6ece..0000000000 --- a/scripts/clib/clib_path.txt +++ /dev/null @@ -1 +0,0 @@ -../../../cpython \ No newline at end of file diff --git a/scripts/clib/clib_test.py b/scripts/clib/clib_test.py new file mode 100644 index 0000000000..1a6547fbf7 --- /dev/null +++ b/scripts/clib/clib_test.py @@ -0,0 +1,79 @@ +import os +import argparse +import shutil +import subprocess + +def main(): + parser = argparse.ArgumentParser(description="Test cpython library") + parser.add_argument("-cp", nargs=1, default="../../../cpython", required=False, help="Local cpython path.") + parser.add_argument("-rp", nargs=1, default="../..", required=False, help="Local RustPython path.") + args = vars(parser.parse_args()) + CPYTHONPATH = args["cp"] + RPYTHONPATH = args["rp"] + + if isinstance(CPYTHONPATH, list): + CPYTHONPATH = CPYTHONPATH[0] + if isinstance(RPYTHONPATH, list): + RPYTHONPATH = RPYTHONPATH[0] + + liblst = list() + + with open("clib_list.txt", 'r') as f: + for line in f.readlines(): + line = line[:line.find('#')] + line = line.strip() + if line: + liblst.append(line) + + for lib in liblst: + lib_exist = False + test_exist = False + test_do = True + message = [f"{lib}:"] + + if os.path.isfile(f"{RPYTHONPATH}/Lib/{lib}.py"): + lib_exist = True + os.rename(f"{RPYTHONPATH}/Lib/{lib}.py", f"{RPYTHONPATH}/Lib/{lib}_tmp_mv.py") + if os.path.isfile(f"{RPYTHONPATH}/Lib/test/test_{lib}.py"): + test_exist = True + os.rename(f"{RPYTHONPATH}/Lib/test/test_{lib}.py", f"{RPYTHONPATH}/Lib/test/test_{lib}_tmp_mv.py") + + if os.path.isfile(f"{CPYTHONPATH}/Lib/{lib}.py"): + shutil.copyfile(f"{CPYTHONPATH}/Lib/{lib}.py", f"{RPYTHONPATH}/Lib/{lib}.py") + else: + test_do = False + message.append(f"No cpython/Lib/{lib}.py") + + if os.path.isfile(f"{CPYTHONPATH}/Lib/test/test_{lib}.py"): + shutil.copyfile(f"{CPYTHONPATH}/Lib/test/test_{lib}.py", f"{RPYTHONPATH}/Lib/test/test_{lib}.py") + else: + test_do = False + message.append(f"No cpython/Lib/test/test_{lib}.py") + + if test_do: + result = subprocess.run(["cargo", "run", "-q", f"{RPYTHONPATH}/Lib/test/test_{lib}.py"], stdout=subprocess.PIPE) + result = result.stdout.decode("utf-8") + + if "OK" in result: + message.append("OK") + else: + message.append("Failed") + + if lib_exist: + os.rename(f"{RPYTHONPATH}/Lib/{lib}_tmp_mv.py", f"{RPYTHONPATH}/Lib/{lib}.py") + elif os.path.isfile(f"{RPYTHONPATH}/Lib/{lib}.py"): + os.remove(f"{RPYTHONPATH}/Lib/{lib}.py") + + if test_exist: + os.rename(f"{RPYTHONPATH}/Lib/test/test_{lib}_tmp_mv.py", f"{RPYTHONPATH}/Lib/test/test_{lib}.py") + elif os.path.isfile(f"{RPYTHONPATH}/Lib/test/test_{lib}.py"): + os.remove(f"{RPYTHONPATH}/Lib/test/test_{lib}.py") + + message.append('\n') + message = ' '.join(message) + + with open("clib_out.txt", "a") as f: + f.write(message) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/clib/clib_test_all.sh b/scripts/clib/clib_test_all.sh deleted file mode 100644 index d73e486c89..0000000000 --- a/scripts/clib/clib_test_all.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -CPYTHONPATH=$(cat clib_path.txt) -RPYTHONPATH="../.." - -cp -r "${RPYTHONPATH}/Lib" "${RPYTHONPATH}/LibTmp" - -index=1 - -while : -do - token=$(sed -n ${index}p clib_list.txt | xargs) - if [ -z $token ] || [ ${token::1} == "#" ]; then - ((index++)) - continue - fi - if [ $token == "EOF" ]; then - break - fi - - if [ -f "${CPYTHONPATH}/Lib/${token}.py" ]; then - cp "${CPYTHONPATH}/Lib/${token}.py" "${RPYTHONPATH}/Lib/${token}.py" - fi - - if [ -f "${CPYTHONPATH}/Lib/test/test_${token}.py" ]; then - cp "${CPYTHONPATH}/Lib/test/test_${token}.py" "${RPYTHONPATH}/Lib/test/test_${token}.py" - fi - - ((index++)) -done - -cargo build -r - -if [ -f "clib_out.txt" ]; then - rm "clib_out.txt" -fi - -touch "clib_out.txt" - -index=1 - -while : -do - token=$(sed -n ${index}p clib_list.txt | xargs) - if [ -z $token ] || [ ${token::1} == "#" ]; then - ((index++)) - continue - fi - if [ $token == "EOF" ]; then - break - fi - - message="${token}:" - - if [ -f "${RPYTHONPATH}/Lib/test/test_${token}.py" ]; then - test=$(cargo run -q -r "${RPYTHONPATH}/Lib/test/test_${token}.py" -q 2>&1 >/dev/null | grep "OK") - - if [ ! -z "${test}" ] ; then - message="${message} OK" - fi - if [ -z "${test}" ] ; then - message="${message} FAILED" - fi - fi - if [ ! -f "${RPYTHONPATH}/Lib/test/test_${token}.py" ]; then - message="${message} NOTEST" - fi - - echo $message >> "clib_out.txt" - - ((index++)) -done - -rm -r "${RPYTHONPATH}/Lib" -mv -r "${RPYTHONPATH}/LibTmp" "${RPYTHONPATH}/Lib" \ No newline at end of file diff --git a/scripts/clib/clib_test_one.sh b/scripts/clib/clib_test_one.sh deleted file mode 100644 index cb139be522..0000000000 --- a/scripts/clib/clib_test_one.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -CPYTHONPATH=$(cat clib_path.txt) -RPYTHONPATH="../.." - -if [ -f "clib_out.txt" ]; then - rm "clib_out.txt" -fi - -touch "clib_out.txt" - -index=1 - -while : -do - token=$(sed -n ${index}p clib_list.txt | xargs) - if [ -z $token ] || [ ${token::1} == "#" ]; then - ((index++)) - continue - fi - if [ $token == "EOF" ]; then - break - fi - - lib_exist=false - test_exist=false - test_do=true - message="${token}:" - - if [ -f "${RPYTHONPATH}/Lib/${token}.py" ]; then - lib_exist=true - cp "${RPYTHONPATH}/Lib/${token}.py" "${RPYTHONPATH}/Lib/${token}_tmp_cp.py" - fi - if [ -f "${RPYTHONPATH}/Lib/test/test_${token}.py" ]; then - test_exist=true - cp "${RPYTHONPATH}/Lib/test/test_${token}.py" "${RPYTHONPATH}/Lib/test/test_${token}_tmp_cp.py" - fi - - if [ -f "${CPYTHONPATH}/Lib/${token}.py" ]; then - cp "${CPYTHONPATH}/Lib/${token}.py" "${RPYTHONPATH}/Lib/${token}.py" - fi - if [ ! -f "${CPYTHONPATH}/Lib/${token}.py" ]; then - test_do=false - message="${message} No cpython/Lib/${token}.py" - fi - - if [ -f "${CPYTHONPATH}/Lib/test/test_${token}.py" ]; then - cp "${CPYTHONPATH}/Lib/test/test_${token}.py" "${RPYTHONPATH}/Lib/test/test_${token}.py" - fi - if [ ! -f "${CPYTHONPATH}/Lib/test/test_${token}.py" ]; then - test_do=false - message="${message} No cpython/Lib/test/test_${token}.py" - fi - - if $test_do ; then - test=$(cargo run -q "${RPYTHONPATH}/Lib/test/test_${token}.py" -q 2>&1 >/dev/null | grep "OK") - - if [ ! -z "${test}" ] ; then - message="${message} OK" - fi - if [ -z "${test}" ] ; then - message="${message} FAILED" - fi - fi - - if $lib_exist ; then - mv "${RPYTHONPATH}/Lib/${token}_tmp_cp.py" "${RPYTHONPATH}/Lib/${token}.py" - fi - if ! $lib_exist && [ -f "${RPYTHONPATH}/Lib/${token}.py" ] ; then - rm "${RPYTHONPATH}/Lib/${token}.py" - fi - - if $test_exist ; then - mv "${RPYTHONPATH}/Lib/test/test_${token}_tmp_cp.py" "${RPYTHONPATH}/Lib/test/test_${token}.py" - fi - if ! $test_exist && [ -f "${RPYTHONPATH}/Lib/test/test_${token}.py" ] ; then - rm "${RPYTHONPATH}/Lib/test/test_${token}.py" - fi - - echo $message >> "clib_out.txt" - - ((index++)) -done \ No newline at end of file diff --git a/scripts/clib/readme.md b/scripts/clib/readme.md index b587ed02d1..d159aab127 100644 --- a/scripts/clib/readme.md +++ b/scripts/clib/readme.md @@ -1,11 +1,9 @@ -To use, write down your local cpython path in clib_path.txt, and run, -bash clib_test_one.sh - +To use, open cmd and run clib_test.py with args through python and check the result in clib_out.txt -The script will try to test every component in clib_list.txt, where, +The script assumes that the script is being run from RustPython/scripts/clib, +and that both RustPython and cpython project directories are located under a same parent directory, aka that they are siblings -clib_test_one.sh : test each component individually -clib_test_all.sh : test every component simultaneously +If either of those assumptions are false, then you must provide a correct path when running clib_test.py -Use of clib_test_all.sh is discouraged, as one failing component can lead to cascading failure of otherwise non-failing components \ No newline at end of file +The script will try to test every component in clib_list.txt \ No newline at end of file From ddbcad92121ffa313249e8f85f2c51331cd73eb8 Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Tue, 25 Apr 2023 10:59:15 +0900 Subject: [PATCH 11/17] Make script to use append for saving output in clib_out.txt --- scripts/clib/clib_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/clib/clib_test.py b/scripts/clib/clib_test.py index 1a6547fbf7..f0bb8a4765 100644 --- a/scripts/clib/clib_test.py +++ b/scripts/clib/clib_test.py @@ -25,6 +25,8 @@ def main(): if line: liblst.append(line) + open("clib_out.txt", 'w').close() + for lib in liblst: lib_exist = False test_exist = False From 94304bbe1e0d6d2143cce1cbeb676f6c128eccde Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Tue, 25 Apr 2023 11:08:06 +0900 Subject: [PATCH 12/17] Fix bug where stderr was not being captured --- scripts/clib/clib_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/clib/clib_test.py b/scripts/clib/clib_test.py index f0bb8a4765..fdbcaf2648 100644 --- a/scripts/clib/clib_test.py +++ b/scripts/clib/clib_test.py @@ -53,7 +53,7 @@ def main(): message.append(f"No cpython/Lib/test/test_{lib}.py") if test_do: - result = subprocess.run(["cargo", "run", "-q", f"{RPYTHONPATH}/Lib/test/test_{lib}.py"], stdout=subprocess.PIPE) + result = subprocess.run(["cargo", "run", "-q", f"{RPYTHONPATH}/Lib/test/test_{lib}.py"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) result = result.stdout.decode("utf-8") if "OK" in result: From 2456d0e20408421ddf06913061384e0f490ca49c Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Tue, 25 Apr 2023 11:13:37 +0900 Subject: [PATCH 13/17] Restore output --- scripts/clib/clib_out.txt | 241 ++++++++++++++++++++++++-------------- 1 file changed, 153 insertions(+), 88 deletions(-) diff --git a/scripts/clib/clib_out.txt b/scripts/clib/clib_out.txt index 77412d6d0e..6b31ee6b62 100644 --- a/scripts/clib/clib_out.txt +++ b/scripts/clib/clib_out.txt @@ -1,88 +1,153 @@ -__future__: Failed -__hello__: No cpython/Lib/test/test___hello__.py -_collections_abc: No cpython/Lib/test/test__collections_abc.py -_compat_pickle: No cpython/Lib/test/test__compat_pickle.py -_compression: No cpython/Lib/test/test__compression.py -_markupbase: No cpython/Lib/test/test__markupbase.py -_osx_support: Failed -_py_abc: No cpython/Lib/test/test__py_abc.py -_pycodecs: No cpython/Lib/_pycodecs.py No cpython/Lib/test/test__pycodecs.py -_pydecimal: No cpython/Lib/test/test__pydecimal.py -_pyio: No cpython/Lib/test/test__pyio.py -_sitebuiltins: No cpython/Lib/test/test__sitebuiltins.py -_threading_local: No cpython/Lib/test/test__threading_local.py -_weakrefset: No cpython/Lib/test/test__weakrefset.py -abc: Failed -aifc: Failed -antigravity: No cpython/Lib/test/test_antigravity.py -argparse: Failed -ast: Failed -asynchat: Failed -asyncore: Failed -base64: Failed -bdb: Failed -binhex: No cpython/Lib/binhex.py No cpython/Lib/test/test_binhex.py -bisect: Failed -bz2: Failed -calendar: Failed -cgi: Failed -cgitb: Failed -chunk: No cpython/Lib/test/test_chunk.py -cmd: Failed -code: Failed -codecs: Failed -codeop: Failed -colorsys: Failed -compileall: Failed -configparser: Failed -contextlib: Failed -contextvars: No cpython/Lib/test/test_contextvars.py -copy: Failed -copyreg: Failed -csv: Failed -dataclasses: Failed -datetime: Failed -decimal: Failed -difflib: Failed -dis: Failed -doctest: Failed -enum: Failed -filecmp: Failed -fileinput: Failed -fnmatch: Failed -formatter: No cpython/Lib/formatter.py No cpython/Lib/test/test_formatter.py -fractions: Failed -ftplib: Failed -functools: Failed -gc: No cpython/Lib/gc.py -genericpath: Failed -getopt: Failed -getpass: Failed -gettext: Failed -glob: Failed -graphlib: Failed -gzip: Failed -hashlib: Failed -heapq: Failed -hmac: Failed -imghdr: Failed -imp: Failed -inspect: Failed -io: Failed -ipaddress: Failed -keyword: Failed -linecache: Failed -locale: Failed -mailbox: Failed -mimetypes: Failed -netrc: OK -nntplib: Failed -ntpath: Failed -nturl2path: No cpython/Lib/test/test_nturl2path.py -numbers: No cpython/Lib/test/test_numbers.py -opcode: No cpython/Lib/test/test_opcode.py -operator: Failed -optparse: Failed -os: Failed -pathlib: Failed -pdb: Failed +__future__: OK +__hello__: No cpython/Lib/test/test___hello__.py +_collections_abc: No cpython/Lib/test/test__collections_abc.py +_compat_pickle: No cpython/Lib/test/test__compat_pickle.py +_compression: No cpython/Lib/test/test__compression.py +_markupbase: No cpython/Lib/test/test__markupbase.py +_osx_support: OK +_py_abc: No cpython/Lib/test/test__py_abc.py +_pycodecs: No cpython/Lib/_pycodecs.py No cpython/Lib/test/test__pycodecs.py +_pydecimal: No cpython/Lib/test/test__pydecimal.py +_pyio: No cpython/Lib/test/test__pyio.py +_sitebuiltins: No cpython/Lib/test/test__sitebuiltins.py +_threading_local: No cpython/Lib/test/test__threading_local.py +_weakrefset: No cpython/Lib/test/test__weakrefset.py +abc: FAILED +aifc: FAILED +antigravity: No cpython/Lib/test/test_antigravity.py +argparse: OK +ast: FAILED +asynchat: FAILED +asyncore: FAILED +base64: OK +bdb: FAILED +binhex: No cpython/Lib/binhex.py No cpython/Lib/test/test_binhex.py +bisect: OK +bz2: FAILED +calendar: OK +cgi: FAILED +cgitb: FAILED +chunk: No cpython/Lib/test/test_chunk.py +cmd: OK +code: FAILED +codecs: FAILED +codeop: FAILED +colorsys: OK +compileall: FAILED +configparser: OK +contextlib: FAILED +contextvars: No cpython/Lib/test/test_contextvars.py +copy: FAILED +copyreg: OK +csv: FAILED +dataclasses: FAILED +datetime: FAILED +decimal: FAILED +difflib: FAILED +dis: FAILED +doctest: FAILED +enum: FAILED +filecmp: OK +fileinput: FAILED +fnmatch: FAILED +formatter: No cpython/Lib/formatter.py No cpython/Lib/test/test_formatter.py +fractions: FAILED +ftplib: OK +functools: FAILED +gc: No cpython/Lib/gc.py +genericpath: OK +getopt: OK +getpass: OK +gettext: OK +glob: OK +graphlib: OK +gzip: OK +hashlib: FAILED +heapq: OK +hmac: OK +imghdr: FAILED +imp: FAILED +inspect: FAILED +io: FAILED +ipaddress: OK +keyword: OK +linecache: FAILED +locale: OK +mailbox: FAILED +mimetypes: OK +netrc: FAILED +nntplib: FAILED +ntpath: FAILED +nturl2path: No cpython/Lib/test/test_nturl2path.py +numbers: No cpython/Lib/test/test_numbers.py +opcode: No cpython/Lib/test/test_opcode.py +operator: OK +optparse: OK +os: FAILED +pathlib: FAILED +pdb: FAILED +pickle: FAILED +pickletools: FAILED +pkgutil: FAILED +platform: OK +plistlib: FAILED +posixpath: FAILED +pprint: FAILED +pty: FAILED +py_compile: FAILED +pydoc: FAILED +queue: OK +quopri: OK +random: FAILED +re: No cpython/Lib/re.py +reprlib: FAILED +rlcompleter: FAILED +runpy: FAILED +sched: OK +secrets: OK +selectors: OK +shelve: FAILED +shlex: FAILED +shutil: FAILED +site: FAILED +smtpd: FAILED +smtplib: FAILED +sndhdr: FAILED +socket: FAILED +socketserver: OK +sre_compile: No cpython/Lib/test/test_sre_compile.py +sre_constants: No cpython/Lib/test/test_sre_constants.py +sre_parse: No cpython/Lib/test/test_sre_parse.py +ssl: FAILED +stat: FAILED +statistics: FAILED +string: OK +stringprep: FAILED +struct: FAILED +subprocess: FAILED +sunau: FAILED +sysconfig: FAILED +tabnanny: FAILED +tarfile: FAILED +telnetlib: FAILED +tempfile: FAILED +textwrap: OK +this: No cpython/Lib/test/test_this.py +threading: FAILED +timeit: FAILED +token: No cpython/Lib/test/test_token.py +tokenize: FAILED +trace: FAILED +traceback: FAILED +tty: No cpython/Lib/test/test_tty.py +types: FAILED +typing: FAILED +uu: FAILED +uuid: FAILED +warnings: No cpython/Lib/test/test_warnings.py +weakref: FAILED +webbrowser: OK +xdrlib: FAILED +zipapp: FAILED +zipfile: FAILED +zipimport: OK \ No newline at end of file From 975e3b8927d213cac8ca859b4731ddeac089856c Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Tue, 25 Apr 2023 16:10:56 +0900 Subject: [PATCH 14/17] Ran the new python script --- scripts/clib/clib_out.txt | 306 +++++++++++++++++++------------------- 1 file changed, 153 insertions(+), 153 deletions(-) diff --git a/scripts/clib/clib_out.txt b/scripts/clib/clib_out.txt index 6b31ee6b62..1bcdc182d1 100644 --- a/scripts/clib/clib_out.txt +++ b/scripts/clib/clib_out.txt @@ -1,153 +1,153 @@ -__future__: OK -__hello__: No cpython/Lib/test/test___hello__.py -_collections_abc: No cpython/Lib/test/test__collections_abc.py -_compat_pickle: No cpython/Lib/test/test__compat_pickle.py -_compression: No cpython/Lib/test/test__compression.py -_markupbase: No cpython/Lib/test/test__markupbase.py -_osx_support: OK -_py_abc: No cpython/Lib/test/test__py_abc.py -_pycodecs: No cpython/Lib/_pycodecs.py No cpython/Lib/test/test__pycodecs.py -_pydecimal: No cpython/Lib/test/test__pydecimal.py -_pyio: No cpython/Lib/test/test__pyio.py -_sitebuiltins: No cpython/Lib/test/test__sitebuiltins.py -_threading_local: No cpython/Lib/test/test__threading_local.py -_weakrefset: No cpython/Lib/test/test__weakrefset.py -abc: FAILED -aifc: FAILED -antigravity: No cpython/Lib/test/test_antigravity.py -argparse: OK -ast: FAILED -asynchat: FAILED -asyncore: FAILED -base64: OK -bdb: FAILED -binhex: No cpython/Lib/binhex.py No cpython/Lib/test/test_binhex.py -bisect: OK -bz2: FAILED -calendar: OK -cgi: FAILED -cgitb: FAILED -chunk: No cpython/Lib/test/test_chunk.py -cmd: OK -code: FAILED -codecs: FAILED -codeop: FAILED -colorsys: OK -compileall: FAILED -configparser: OK -contextlib: FAILED -contextvars: No cpython/Lib/test/test_contextvars.py -copy: FAILED -copyreg: OK -csv: FAILED -dataclasses: FAILED -datetime: FAILED -decimal: FAILED -difflib: FAILED -dis: FAILED -doctest: FAILED -enum: FAILED -filecmp: OK -fileinput: FAILED -fnmatch: FAILED -formatter: No cpython/Lib/formatter.py No cpython/Lib/test/test_formatter.py -fractions: FAILED -ftplib: OK -functools: FAILED -gc: No cpython/Lib/gc.py -genericpath: OK -getopt: OK -getpass: OK -gettext: OK -glob: OK -graphlib: OK -gzip: OK -hashlib: FAILED -heapq: OK -hmac: OK -imghdr: FAILED -imp: FAILED -inspect: FAILED -io: FAILED -ipaddress: OK -keyword: OK -linecache: FAILED -locale: OK -mailbox: FAILED -mimetypes: OK -netrc: FAILED -nntplib: FAILED -ntpath: FAILED -nturl2path: No cpython/Lib/test/test_nturl2path.py -numbers: No cpython/Lib/test/test_numbers.py -opcode: No cpython/Lib/test/test_opcode.py -operator: OK -optparse: OK -os: FAILED -pathlib: FAILED -pdb: FAILED -pickle: FAILED -pickletools: FAILED -pkgutil: FAILED -platform: OK -plistlib: FAILED -posixpath: FAILED -pprint: FAILED -pty: FAILED -py_compile: FAILED -pydoc: FAILED -queue: OK -quopri: OK -random: FAILED -re: No cpython/Lib/re.py -reprlib: FAILED -rlcompleter: FAILED -runpy: FAILED -sched: OK -secrets: OK -selectors: OK -shelve: FAILED -shlex: FAILED -shutil: FAILED -site: FAILED -smtpd: FAILED -smtplib: FAILED -sndhdr: FAILED -socket: FAILED -socketserver: OK -sre_compile: No cpython/Lib/test/test_sre_compile.py -sre_constants: No cpython/Lib/test/test_sre_constants.py -sre_parse: No cpython/Lib/test/test_sre_parse.py -ssl: FAILED -stat: FAILED -statistics: FAILED -string: OK -stringprep: FAILED -struct: FAILED -subprocess: FAILED -sunau: FAILED -sysconfig: FAILED -tabnanny: FAILED -tarfile: FAILED -telnetlib: FAILED -tempfile: FAILED -textwrap: OK -this: No cpython/Lib/test/test_this.py -threading: FAILED -timeit: FAILED -token: No cpython/Lib/test/test_token.py -tokenize: FAILED -trace: FAILED -traceback: FAILED -tty: No cpython/Lib/test/test_tty.py -types: FAILED -typing: FAILED -uu: FAILED -uuid: FAILED -warnings: No cpython/Lib/test/test_warnings.py -weakref: FAILED -webbrowser: OK -xdrlib: FAILED -zipapp: FAILED -zipfile: FAILED -zipimport: OK \ No newline at end of file +__future__: OK +__hello__: No cpython/Lib/test/test___hello__.py +_collections_abc: No cpython/Lib/test/test__collections_abc.py +_compat_pickle: No cpython/Lib/test/test__compat_pickle.py +_compression: No cpython/Lib/test/test__compression.py +_markupbase: No cpython/Lib/test/test__markupbase.py +_osx_support: OK +_py_abc: No cpython/Lib/test/test__py_abc.py +_pycodecs: No cpython/Lib/_pycodecs.py No cpython/Lib/test/test__pycodecs.py +_pydecimal: No cpython/Lib/test/test__pydecimal.py +_pyio: No cpython/Lib/test/test__pyio.py +_sitebuiltins: No cpython/Lib/test/test__sitebuiltins.py +_threading_local: No cpython/Lib/test/test__threading_local.py +_weakrefset: No cpython/Lib/test/test__weakrefset.py +abc: Failed +aifc: Failed +antigravity: No cpython/Lib/test/test_antigravity.py +argparse: OK +ast: Failed +asynchat: Failed +asyncore: Failed +base64: OK +bdb: Failed +binhex: No cpython/Lib/binhex.py No cpython/Lib/test/test_binhex.py +bisect: OK +bz2: Failed +calendar: OK +cgi: Failed +cgitb: Failed +chunk: No cpython/Lib/test/test_chunk.py +cmd: OK +code: Failed +codecs: Failed +codeop: Failed +colorsys: OK +compileall: Failed +configparser: OK +contextlib: Failed +contextvars: No cpython/Lib/test/test_contextvars.py +copy: Failed +copyreg: OK +csv: Failed +dataclasses: Failed +datetime: Failed +decimal: Failed +difflib: Failed +dis: Failed +doctest: Failed +enum: Failed +filecmp: OK +fileinput: Failed +fnmatch: Failed +formatter: No cpython/Lib/formatter.py No cpython/Lib/test/test_formatter.py +fractions: Failed +ftplib: OK +functools: Failed +gc: No cpython/Lib/gc.py +genericpath: OK +getopt: OK +getpass: OK +gettext: OK +glob: OK +graphlib: OK +gzip: OK +hashlib: Failed +heapq: OK +hmac: OK +imghdr: Failed +imp: Failed +inspect: Failed +io: Failed +ipaddress: OK +keyword: OK +linecache: Failed +locale: OK +mailbox: Failed +mimetypes: OK +netrc: OK +nntplib: Failed +ntpath: Failed +nturl2path: No cpython/Lib/test/test_nturl2path.py +numbers: No cpython/Lib/test/test_numbers.py +opcode: No cpython/Lib/test/test_opcode.py +operator: OK +optparse: OK +os: Failed +pathlib: Failed +pdb: Failed +pickle: Failed +pickletools: Failed +pkgutil: Failed +platform: OK +plistlib: Failed +posixpath: Failed +pprint: Failed +pty: Failed +py_compile: Failed +pydoc: Failed +queue: OK +quopri: OK +random: Failed +re: No cpython/Lib/re.py +reprlib: Failed +rlcompleter: Failed +runpy: Failed +sched: OK +secrets: OK +selectors: OK +shelve: Failed +shlex: Failed +shutil: Failed +site: Failed +smtpd: Failed +smtplib: Failed +sndhdr: Failed +socket: Failed +socketserver: OK +sre_compile: No cpython/Lib/test/test_sre_compile.py +sre_constants: No cpython/Lib/test/test_sre_constants.py +sre_parse: No cpython/Lib/test/test_sre_parse.py +ssl: Failed +stat: Failed +statistics: Failed +string: OK +stringprep: Failed +struct: Failed +subprocess: Failed +sunau: Failed +sysconfig: Failed +tabnanny: Failed +tarfile: Failed +telnetlib: Failed +tempfile: Failed +textwrap: OK +this: No cpython/Lib/test/test_this.py +threading: Failed +timeit: Failed +token: No cpython/Lib/test/test_token.py +tokenize: Failed +trace: Failed +traceback: Failed +tty: No cpython/Lib/test/test_tty.py +types: Failed +typing: Failed +uu: Failed +uuid: Failed +warnings: No cpython/Lib/test/test_warnings.py +weakref: Failed +webbrowser: OK +xdrlib: Failed +zipapp: Failed +zipfile: Failed +zipimport: OK From e5d85e1e9544532d03edf34e3dbc138b050629b1 Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Wed, 26 Apr 2023 12:38:27 +0900 Subject: [PATCH 15/17] Apply feedback --- scripts/clib/clib_list.txt | 2 +- scripts/clib/clib_test.py | 170 +++++++++++++++++++++++-------------- 2 files changed, 109 insertions(+), 63 deletions(-) diff --git a/scripts/clib/clib_list.txt b/scripts/clib/clib_list.txt index 28ca9e585a..811bbb977d 100644 --- a/scripts/clib/clib_list.txt +++ b/scripts/clib/clib_list.txt @@ -1,4 +1,4 @@ -__future__ +#__future__ __hello__ _collections_abc _compat_pickle diff --git a/scripts/clib/clib_test.py b/scripts/clib/clib_test.py index fdbcaf2648..de3c50ceba 100644 --- a/scripts/clib/clib_test.py +++ b/scripts/clib/clib_test.py @@ -1,81 +1,127 @@ -import os import argparse +import os import shutil import subprocess -def main(): - parser = argparse.ArgumentParser(description="Test cpython library") - parser.add_argument("-cp", nargs=1, default="../../../cpython", required=False, help="Local cpython path.") - parser.add_argument("-rp", nargs=1, default="../..", required=False, help="Local RustPython path.") - args = vars(parser.parse_args()) - CPYTHONPATH = args["cp"] - RPYTHONPATH = args["rp"] +from dataclasses import dataclass +from pathlib import Path +from typing import List - if isinstance(CPYTHONPATH, list): - CPYTHONPATH = CPYTHONPATH[0] - if isinstance(RPYTHONPATH, list): - RPYTHONPATH = RPYTHONPATH[0] +@dataclass +class LibEntry: + name: str + lib_exist: bool # Checks whether the library file existed in RustPython/Lib prior to testing + test_exist: bool # Checks whether the test file existed in RustPython/Lib/test prior to testing + test_do: bool # Checks if both the library file and the test file existed in cpython and therefore is possible to test + test_ok: bool # Checks if the test returned OK or not + path_cpython_lib: str + path_cpython_test: str + path_rpython_lib: str + path_rpython_test: str + path_rpython_tmp_lib: str + path_rpython_tmp_test: str - liblst = list() + def __init__(self, name: str, CPYTHON_PATH: str, RPYTHON_PATH: str): + self.name = name - with open("clib_list.txt", 'r') as f: - for line in f.readlines(): - line = line[:line.find('#')] - line = line.strip() - if line: - liblst.append(line) + self.path_cpython_lib = os.path.join(CPYTHON_PATH, "Lib", f"{self.name}.py") + self.path_rpython_lib = os.path.join(RPYTHON_PATH, "Lib", f"{self.name}.py") + self.path_rpython_tmp_lib = os.path.join(RPYTHON_PATH, "LibTmp", f"{self.name}.py") - open("clib_out.txt", 'w').close() + self.path_cpython_test = os.path.join(CPYTHON_PATH, "Lib", "test", f"test_{self.name}.py") + self.path_rpython_test = os.path.join(RPYTHON_PATH, "Lib", "test", f"test_{self.name}.py") + self.path_rpython_tmp_test = os.path.join(RPYTHON_PATH, "LibTmp", "test", f"test_{self.name}.py") - for lib in liblst: - lib_exist = False - test_exist = False - test_do = True - message = [f"{lib}:"] + self.lib_exist = os.path.isfile(self.path_rpython_lib) + self.test_exist = os.path.isfile(self.path_rpython_test) + self.test_do = os.path.isfile(self.path_cpython_lib) and os.path.isfile(self.path_cpython_test) - if os.path.isfile(f"{RPYTHONPATH}/Lib/{lib}.py"): - lib_exist = True - os.rename(f"{RPYTHONPATH}/Lib/{lib}.py", f"{RPYTHONPATH}/Lib/{lib}_tmp_mv.py") - if os.path.isfile(f"{RPYTHONPATH}/Lib/test/test_{lib}.py"): - test_exist = True - os.rename(f"{RPYTHONPATH}/Lib/test/test_{lib}.py", f"{RPYTHONPATH}/Lib/test/test_{lib}_tmp_mv.py") + def run(self, CPYTHON_PATH: str, RPYTHON_PATH: str): + if self.test_do: + shutil.copyfile(self.path_cpython_lib, self.path_rpython_lib) + shutil.copyfile(self.path_cpython_test, self.path_rpython_test) - if os.path.isfile(f"{CPYTHONPATH}/Lib/{lib}.py"): - shutil.copyfile(f"{CPYTHONPATH}/Lib/{lib}.py", f"{RPYTHONPATH}/Lib/{lib}.py") - else: - test_do = False - message.append(f"No cpython/Lib/{lib}.py") - - if os.path.isfile(f"{CPYTHONPATH}/Lib/test/test_{lib}.py"): - shutil.copyfile(f"{CPYTHONPATH}/Lib/test/test_{lib}.py", f"{RPYTHONPATH}/Lib/test/test_{lib}.py") - else: - test_do = False - message.append(f"No cpython/Lib/test/test_{lib}.py") - - if test_do: - result = subprocess.run(["cargo", "run", "-q", f"{RPYTHONPATH}/Lib/test/test_{lib}.py"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + result = subprocess.run( + ["cargo", "run", "-q", self.path_rpython_test], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) result = result.stdout.decode("utf-8") - if "OK" in result: + self.test_ok = "OK" in result + + if self.lib_exist: + os.rename(self.path_rpython_tmp_lib, self.path_rpython_lib) + else: + os.remove(self.path_rpython_lib) + + if self.test_exist: + os.rename(self.path_rpython_tmp_test, self.path_rpython_test) + else: + os.remove(self.path_rpython_test) + + def to_string(self): + message = [f"{self.name}:"] + + if self.test_do: + if self.test_ok: message.append("OK") else: message.append("Failed") - - if lib_exist: - os.rename(f"{RPYTHONPATH}/Lib/{lib}_tmp_mv.py", f"{RPYTHONPATH}/Lib/{lib}.py") - elif os.path.isfile(f"{RPYTHONPATH}/Lib/{lib}.py"): - os.remove(f"{RPYTHONPATH}/Lib/{lib}.py") - - if test_exist: - os.rename(f"{RPYTHONPATH}/Lib/test/test_{lib}_tmp_mv.py", f"{RPYTHONPATH}/Lib/test/test_{lib}.py") - elif os.path.isfile(f"{RPYTHONPATH}/Lib/test/test_{lib}.py"): - os.remove(f"{RPYTHONPATH}/Lib/test/test_{lib}.py") - - message.append('\n') - message = ' '.join(message) - - with open("clib_out.txt", "a") as f: - f.write(message) + else: + message.append("No cpython lib or test file") + + return ' '.join(message) + +def main(): + CURRENT_PATH = Path(__file__) + + parser = argparse.ArgumentParser(description="Test cpython library") + parser.add_argument( + "--cpython", + nargs=1, + default=os.path.join( + CURRENT_PATH.parents[3], + "cpython" + ), + required=False, + help="Local cpython path." + ) + parser.add_argument( + "--rustpython", + nargs=1, + default=CURRENT_PATH.parents[2], + required=False, + help="Local RustPython path." + ) + args = vars(parser.parse_args()) + CPYTHON_PATH = args["cpython"] + RPYTHON_PATH = args["rustpython"] + + if isinstance(CPYTHON_PATH, list): + CPYTHON_PATH = CPYTHON_PATH[0] + if isinstance(RPYTHON_PATH, list): + RPYTHON_PATH = RPYTHON_PATH[0] + + shutil.copytree( + os.path.join(RPYTHON_PATH, "Lib"), + os.path.join(RPYTHON_PATH, "LibTmp") + ) + + library_list: List[LibEntry] = [] + + with open(os.path.join(CURRENT_PATH.parent, "clib_list.txt"), 'r') as f: + for line in f: + line = line.split('#')[0] + line = line.strip() + if line: + library_list.append(LibEntry(line, CPYTHON_PATH, RPYTHON_PATH)) + + for entry in library_list: + entry.run(CPYTHON_PATH, RPYTHON_PATH) + print(entry.to_string()) + + shutil.rmtree(os.path.join(RPYTHON_PATH, "LibTmp")) if __name__ == "__main__": main() \ No newline at end of file From 6fc49f17f019276a02107d6a2a849535612fe85e Mon Sep 17 00:00:00 2001 From: MegasKomnenos Date: Fri, 28 Apr 2023 07:08:38 +0900 Subject: [PATCH 16/17] Applied feedback Changed code structure to prevent rebuilding cargo for each iteration of the loop Made the script to no longer edit the lib files directly, and instead used RUSTPYTHONPATH to reference a temporary lib folder --- scripts/clib/clib_test.py | 40 +++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/scripts/clib/clib_test.py b/scripts/clib/clib_test.py index de3c50ceba..c6ca44e509 100644 --- a/scripts/clib/clib_test.py +++ b/scripts/clib/clib_test.py @@ -18,47 +18,48 @@ class LibEntry: path_cpython_test: str path_rpython_lib: str path_rpython_test: str - path_rpython_tmp_lib: str - path_rpython_tmp_test: str + path_tpython_lib: str + path_tpython_test: str def __init__(self, name: str, CPYTHON_PATH: str, RPYTHON_PATH: str): self.name = name self.path_cpython_lib = os.path.join(CPYTHON_PATH, "Lib", f"{self.name}.py") self.path_rpython_lib = os.path.join(RPYTHON_PATH, "Lib", f"{self.name}.py") - self.path_rpython_tmp_lib = os.path.join(RPYTHON_PATH, "LibTmp", f"{self.name}.py") + self.path_tpython_lib = os.path.join(RPYTHON_PATH, "LibTest", f"{self.name}.py") self.path_cpython_test = os.path.join(CPYTHON_PATH, "Lib", "test", f"test_{self.name}.py") self.path_rpython_test = os.path.join(RPYTHON_PATH, "Lib", "test", f"test_{self.name}.py") - self.path_rpython_tmp_test = os.path.join(RPYTHON_PATH, "LibTmp", "test", f"test_{self.name}.py") + self.path_tpython_test = os.path.join(RPYTHON_PATH, "LibTest", "test", f"test_{self.name}.py") - self.lib_exist = os.path.isfile(self.path_rpython_lib) - self.test_exist = os.path.isfile(self.path_rpython_test) + self.lib_exist = os.path.isfile(self.path_tpython_lib) + self.test_exist = os.path.isfile(self.path_tpython_test) self.test_do = os.path.isfile(self.path_cpython_lib) and os.path.isfile(self.path_cpython_test) - def run(self, CPYTHON_PATH: str, RPYTHON_PATH: str): + def run(self, RUSTEXECPATH: str): if self.test_do: - shutil.copyfile(self.path_cpython_lib, self.path_rpython_lib) - shutil.copyfile(self.path_cpython_test, self.path_rpython_test) + shutil.copyfile(self.path_cpython_lib, self.path_tpython_lib) + shutil.copyfile(self.path_cpython_test, self.path_tpython_test) result = subprocess.run( - ["cargo", "run", "-q", self.path_rpython_test], + [RUSTEXECPATH, "-q", self.path_tpython_test], stdout=subprocess.PIPE, - stderr=subprocess.STDOUT + stderr=subprocess.STDOUT, + env={"RUSTPYTHONPATH": "LibTest"} ) result = result.stdout.decode("utf-8") self.test_ok = "OK" in result if self.lib_exist: - os.rename(self.path_rpython_tmp_lib, self.path_rpython_lib) + shutil.copyfile(self.path_rpython_lib, self.path_tpython_lib) else: - os.remove(self.path_rpython_lib) + os.remove(self.path_tpython_lib) if self.test_exist: - os.rename(self.path_rpython_tmp_test, self.path_rpython_test) + shutil.copyfile(self.path_rpython_test, self.path_tpython_test) else: - os.remove(self.path_rpython_test) + os.remove(self.path_tpython_test) def to_string(self): message = [f"{self.name}:"] @@ -105,7 +106,7 @@ def main(): shutil.copytree( os.path.join(RPYTHON_PATH, "Lib"), - os.path.join(RPYTHON_PATH, "LibTmp") + os.path.join(RPYTHON_PATH, "LibTest") ) library_list: List[LibEntry] = [] @@ -117,11 +118,14 @@ def main(): if line: library_list.append(LibEntry(line, CPYTHON_PATH, RPYTHON_PATH)) + subprocess.run(["cargo", "build", "--release", "-q"]) + REXECPATH = os.path.join(CURRENT_PATH.parents[2], "target", "release", "rustpython") + for entry in library_list: - entry.run(CPYTHON_PATH, RPYTHON_PATH) + entry.run(REXECPATH) print(entry.to_string()) - shutil.rmtree(os.path.join(RPYTHON_PATH, "LibTmp")) + shutil.rmtree(os.path.join(RPYTHON_PATH, "LibTest")) if __name__ == "__main__": main() \ No newline at end of file From de35383c88aa026c38e48a2797520ae8a8a160d4 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 26 Mar 2025 22:26:28 +0900 Subject: [PATCH 17/17] renaming and LibTest cleanup --- .../{clib/readme.md => autocheck/README.md} | 7 +- .../clib_test.py => autocheck/autocheck.py} | 22 ++- .../clib_list.txt => autocheck/targets.txt} | 0 scripts/clib/clib_out.txt | 153 ------------------ 4 files changed, 17 insertions(+), 165 deletions(-) rename scripts/{clib/readme.md => autocheck/README.md} (57%) rename scripts/{clib/clib_test.py => autocheck/autocheck.py} (89%) rename scripts/{clib/clib_list.txt => autocheck/targets.txt} (100%) delete mode 100644 scripts/clib/clib_out.txt diff --git a/scripts/clib/readme.md b/scripts/autocheck/README.md similarity index 57% rename from scripts/clib/readme.md rename to scripts/autocheck/README.md index d159aab127..b7fd50b545 100644 --- a/scripts/clib/readme.md +++ b/scripts/autocheck/README.md @@ -1,9 +1,8 @@ -To use, open cmd and run clib_test.py with args through python -and check the result in clib_out.txt +To use, open cmd and run run_autotest.py The script assumes that the script is being run from RustPython/scripts/clib, and that both RustPython and cpython project directories are located under a same parent directory, aka that they are siblings -If either of those assumptions are false, then you must provide a correct path when running clib_test.py +If either of those assumptions are false, then you must provide a correct path when running run_autotest.py -The script will try to test every component in clib_list.txt \ No newline at end of file +The script will try to test every component in targets.txt \ No newline at end of file diff --git a/scripts/clib/clib_test.py b/scripts/autocheck/autocheck.py similarity index 89% rename from scripts/clib/clib_test.py rename to scripts/autocheck/autocheck.py index c6ca44e509..53738d2894 100644 --- a/scripts/clib/clib_test.py +++ b/scripts/autocheck/autocheck.py @@ -2,11 +2,16 @@ import os import shutil import subprocess +import sys from dataclasses import dataclass from pathlib import Path from typing import List +def subprocess_run(*args, **kwargs): + # print('Process run:', *args, kwargs, file=sys.stderr) + return subprocess.run(*args, **kwargs) + @dataclass class LibEntry: name: str @@ -41,7 +46,7 @@ def run(self, RUSTEXECPATH: str): shutil.copyfile(self.path_cpython_lib, self.path_tpython_lib) shutil.copyfile(self.path_cpython_test, self.path_tpython_test) - result = subprocess.run( + result = subprocess_run( [RUSTEXECPATH, "-q", self.path_tpython_test], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -111,21 +116,22 @@ def main(): library_list: List[LibEntry] = [] - with open(os.path.join(CURRENT_PATH.parent, "clib_list.txt"), 'r') as f: + with open(os.path.join(CURRENT_PATH.parent, "targets.txt"), 'r') as f: for line in f: line = line.split('#')[0] line = line.strip() if line: library_list.append(LibEntry(line, CPYTHON_PATH, RPYTHON_PATH)) - subprocess.run(["cargo", "build", "--release", "-q"]) + subprocess_run(["cargo", "build", "--release", "-q"]) REXECPATH = os.path.join(CURRENT_PATH.parents[2], "target", "release", "rustpython") - for entry in library_list: - entry.run(REXECPATH) - print(entry.to_string()) - - shutil.rmtree(os.path.join(RPYTHON_PATH, "LibTest")) + try: + for entry in library_list: + entry.run(REXECPATH) + print(entry.to_string()) + finally: + shutil.rmtree(os.path.join(RPYTHON_PATH, "LibTest")) if __name__ == "__main__": main() \ No newline at end of file diff --git a/scripts/clib/clib_list.txt b/scripts/autocheck/targets.txt similarity index 100% rename from scripts/clib/clib_list.txt rename to scripts/autocheck/targets.txt diff --git a/scripts/clib/clib_out.txt b/scripts/clib/clib_out.txt deleted file mode 100644 index 1bcdc182d1..0000000000 --- a/scripts/clib/clib_out.txt +++ /dev/null @@ -1,153 +0,0 @@ -__future__: OK -__hello__: No cpython/Lib/test/test___hello__.py -_collections_abc: No cpython/Lib/test/test__collections_abc.py -_compat_pickle: No cpython/Lib/test/test__compat_pickle.py -_compression: No cpython/Lib/test/test__compression.py -_markupbase: No cpython/Lib/test/test__markupbase.py -_osx_support: OK -_py_abc: No cpython/Lib/test/test__py_abc.py -_pycodecs: No cpython/Lib/_pycodecs.py No cpython/Lib/test/test__pycodecs.py -_pydecimal: No cpython/Lib/test/test__pydecimal.py -_pyio: No cpython/Lib/test/test__pyio.py -_sitebuiltins: No cpython/Lib/test/test__sitebuiltins.py -_threading_local: No cpython/Lib/test/test__threading_local.py -_weakrefset: No cpython/Lib/test/test__weakrefset.py -abc: Failed -aifc: Failed -antigravity: No cpython/Lib/test/test_antigravity.py -argparse: OK -ast: Failed -asynchat: Failed -asyncore: Failed -base64: OK -bdb: Failed -binhex: No cpython/Lib/binhex.py No cpython/Lib/test/test_binhex.py -bisect: OK -bz2: Failed -calendar: OK -cgi: Failed -cgitb: Failed -chunk: No cpython/Lib/test/test_chunk.py -cmd: OK -code: Failed -codecs: Failed -codeop: Failed -colorsys: OK -compileall: Failed -configparser: OK -contextlib: Failed -contextvars: No cpython/Lib/test/test_contextvars.py -copy: Failed -copyreg: OK -csv: Failed -dataclasses: Failed -datetime: Failed -decimal: Failed -difflib: Failed -dis: Failed -doctest: Failed -enum: Failed -filecmp: OK -fileinput: Failed -fnmatch: Failed -formatter: No cpython/Lib/formatter.py No cpython/Lib/test/test_formatter.py -fractions: Failed -ftplib: OK -functools: Failed -gc: No cpython/Lib/gc.py -genericpath: OK -getopt: OK -getpass: OK -gettext: OK -glob: OK -graphlib: OK -gzip: OK -hashlib: Failed -heapq: OK -hmac: OK -imghdr: Failed -imp: Failed -inspect: Failed -io: Failed -ipaddress: OK -keyword: OK -linecache: Failed -locale: OK -mailbox: Failed -mimetypes: OK -netrc: OK -nntplib: Failed -ntpath: Failed -nturl2path: No cpython/Lib/test/test_nturl2path.py -numbers: No cpython/Lib/test/test_numbers.py -opcode: No cpython/Lib/test/test_opcode.py -operator: OK -optparse: OK -os: Failed -pathlib: Failed -pdb: Failed -pickle: Failed -pickletools: Failed -pkgutil: Failed -platform: OK -plistlib: Failed -posixpath: Failed -pprint: Failed -pty: Failed -py_compile: Failed -pydoc: Failed -queue: OK -quopri: OK -random: Failed -re: No cpython/Lib/re.py -reprlib: Failed -rlcompleter: Failed -runpy: Failed -sched: OK -secrets: OK -selectors: OK -shelve: Failed -shlex: Failed -shutil: Failed -site: Failed -smtpd: Failed -smtplib: Failed -sndhdr: Failed -socket: Failed -socketserver: OK -sre_compile: No cpython/Lib/test/test_sre_compile.py -sre_constants: No cpython/Lib/test/test_sre_constants.py -sre_parse: No cpython/Lib/test/test_sre_parse.py -ssl: Failed -stat: Failed -statistics: Failed -string: OK -stringprep: Failed -struct: Failed -subprocess: Failed -sunau: Failed -sysconfig: Failed -tabnanny: Failed -tarfile: Failed -telnetlib: Failed -tempfile: Failed -textwrap: OK -this: No cpython/Lib/test/test_this.py -threading: Failed -timeit: Failed -token: No cpython/Lib/test/test_token.py -tokenize: Failed -trace: Failed -traceback: Failed -tty: No cpython/Lib/test/test_tty.py -types: Failed -typing: Failed -uu: Failed -uuid: Failed -warnings: No cpython/Lib/test/test_warnings.py -weakref: Failed -webbrowser: OK -xdrlib: Failed -zipapp: Failed -zipfile: Failed -zipimport: OK