|
| 1 | +""" |
| 2 | +An automated script to mark failures in python test suite. |
| 3 | +It adds @unittest.expectedFailure to the test functions that are failing in RustPython, but not in CPython. |
| 4 | +As well as marking the test with a TODO comment. |
| 5 | +
|
| 6 | +How to use: |
| 7 | +1. Copy a specific test from the CPython repository to the RustPython repository. |
| 8 | +2. Remove all unexpected failures from the test and skip the tests that hang |
| 9 | +3. Run python ./scripts/fix_test.py --test test_venv --path ./Lib/test/test_venv.py or equivalent for the test from the project root. |
| 10 | +4. Ensure that there are no unexpected successes in the test. |
| 11 | +5. Actually fix the test. |
| 12 | +""" |
| 13 | +import argparse |
| 14 | +import ast |
| 15 | +import itertools |
| 16 | +import platform |
| 17 | +from pathlib import Path |
| 18 | + |
| 19 | +def parse_args(): |
| 20 | + parser = argparse.ArgumentParser(description="Fix test.") |
| 21 | + parser.add_argument("--path", type=Path, help="Path to test file") |
| 22 | + parser.add_argument("--force", action="store_true", help="Force modification") |
| 23 | + parser.add_argument("--platform", action="store_true", help="Platform specific failure") |
| 24 | + |
| 25 | + args = parser.parse_args() |
| 26 | + return args |
| 27 | + |
| 28 | +class Test: |
| 29 | + name: str = "" |
| 30 | + path: str = "" |
| 31 | + result: str = "" |
| 32 | + |
| 33 | + def __str__(self): |
| 34 | + return f"Test(name={self.name}, path={self.path}, result={self.result})" |
| 35 | + |
| 36 | +class TestResult: |
| 37 | + tests_result: str = "" |
| 38 | + tests = [] |
| 39 | + stdout = "" |
| 40 | + |
| 41 | + def __str__(self): |
| 42 | + return f"TestResult(tests_result={self.tests_result},tests={len(self.tests)})" |
| 43 | + |
| 44 | + |
| 45 | +def parse_results(result): |
| 46 | + lines = result.stdout.splitlines() |
| 47 | + test_results = TestResult() |
| 48 | + test_results.stdout = result.stdout |
| 49 | + in_test_results = False |
| 50 | + for line in lines: |
| 51 | + if line == "Run tests sequentially": |
| 52 | + in_test_results = True |
| 53 | + elif line.startswith("-----------"): |
| 54 | + in_test_results = False |
| 55 | + if in_test_results and not line.startswith("tests") and not line.startswith("["): |
| 56 | + line = line.split(" ") |
| 57 | + if line != [] and len(line) > 3: |
| 58 | + test = Test() |
| 59 | + test.name = line[0] |
| 60 | + test.path = line[1].strip("(").strip(")") |
| 61 | + test.result = " ".join(line[3:]).lower() |
| 62 | + test_results.tests.append(test) |
| 63 | + else: |
| 64 | + if "== Tests result: " in line: |
| 65 | + res = line.split("== Tests result: ")[1] |
| 66 | + res = res.split(" ")[0] |
| 67 | + test_results.tests_result = res |
| 68 | + return test_results |
| 69 | + |
| 70 | +def path_to_test(path) -> list[str]: |
| 71 | + return path.split(".")[2:] |
| 72 | + |
| 73 | +def modify_test(file: str, test: list[str], for_platform: bool = False) -> str: |
| 74 | + a = ast.parse(file) |
| 75 | + lines = file.splitlines() |
| 76 | + fixture = "@unittest.expectedFailure" |
| 77 | + for node in ast.walk(a): |
| 78 | + if isinstance(node, ast.FunctionDef): |
| 79 | + if node.name == test[-1]: |
| 80 | + assert not for_platform |
| 81 | + indent = " " * node.col_offset |
| 82 | + lines.insert(node.lineno - 1, indent + fixture) |
| 83 | + lines.insert(node.lineno - 1, indent + "# TODO: RUSTPYTHON") |
| 84 | + break |
| 85 | + return "\n".join(lines) |
| 86 | + |
| 87 | +def modify_test_v2(file: str, test: list[str], for_platform: bool = False) -> str: |
| 88 | + a = ast.parse(file) |
| 89 | + lines = file.splitlines() |
| 90 | + fixture = "@unittest.expectedFailure" |
| 91 | + for key, node in ast.iter_fields(a): |
| 92 | + if key == "body": |
| 93 | + for i, n in enumerate(node): |
| 94 | + match n: |
| 95 | + case ast.ClassDef(): |
| 96 | + if len(test) == 2 and test[0] == n.name: |
| 97 | + # look through body for function def |
| 98 | + for i, fn in enumerate(n.body): |
| 99 | + match fn: |
| 100 | + case ast.FunctionDef(): |
| 101 | + if fn.name == test[-1]: |
| 102 | + assert not for_platform |
| 103 | + indent = " " * fn.col_offset |
| 104 | + lines.insert(fn.lineno - 1, indent + fixture) |
| 105 | + lines.insert(fn.lineno - 1, indent + "# TODO: RUSTPYTHON") |
| 106 | + break |
| 107 | + case ast.FunctionDef(): |
| 108 | + if n.name == test[0] and len(test) == 1: |
| 109 | + assert not for_platform |
| 110 | + indent = " " * n.col_offset |
| 111 | + lines.insert(n.lineno - 1, indent + fixture) |
| 112 | + lines.insert(n.lineno - 1, indent + "# TODO: RUSTPYTHON") |
| 113 | + break |
| 114 | + if i > 500: |
| 115 | + exit() |
| 116 | + return "\n".join(lines) |
| 117 | + |
| 118 | +def run_test(test_name): |
| 119 | + print(f"Running test: {test_name}") |
| 120 | + rustpython_location = "./target/release/rustpython" |
| 121 | + import subprocess |
| 122 | + result = subprocess.run([rustpython_location, "-m", "test", "-v", test_name], capture_output=True, text=True) |
| 123 | + return parse_results(result) |
| 124 | + |
| 125 | + |
| 126 | +if __name__ == "__main__": |
| 127 | + args = parse_args() |
| 128 | + test_name = args.path.stem |
| 129 | + tests = run_test(test_name) |
| 130 | <
692E
code class="diff-text syntax-highlighted-line addition">+ f = open(args.path).read() |
| 131 | + for test in tests.tests: |
| 132 | + if test.result == "fail" or test.result == "error": |
| 133 | + print("Modifying test:", test.name) |
| 134 | + f = modify_test_v2(f, path_to_test(test.path), args.platform) |
| 135 | + with open(args.path, "w") as file: |
| 136 | + # TODO: Find validation method, and make --force override it |
| 137 | + file.write(f) |
0 commit comments