From 5148727e060c947a3fc0ebb942e4ce2dc2dad2e7 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Sat, 13 Jul 2024 12:50:17 -0500 Subject: [PATCH 001/115] Create simple warning check tool and add to ubuntu build and test job --- .github/workflows/reusable-ubuntu.yml | 4 +- Tools/build/.warnignore_ubuntu | 3 + Tools/build/check_warnings.py | 181 ++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 Tools/build/.warnignore_ubuntu create mode 100644 Tools/build/check_warnings.py diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 54d7765d159d49..3a6f2b910d0da0 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -74,10 +74,12 @@ jobs: ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 + run: make -j4 &> compiler_output.txt - name: Display build info working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo + - name: Check compiler warnings + run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw diff --git a/Tools/build/.warnignore_ubuntu b/Tools/build/.warnignore_ubuntu new file mode 100644 index 00000000000000..8242c8d17c89fb --- /dev/null +++ b/Tools/build/.warnignore_ubuntu @@ -0,0 +1,3 @@ +# Files listed will be ignored by the compiler warning checker +# for the Ubuntu/build and test job. +# Keep lines sorted lexicographically to help avoid merge conflicts. diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py new file mode 100644 index 00000000000000..b2119b40f3de6e --- /dev/null +++ b/Tools/build/check_warnings.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path +import argparse +import json + + +def extract_json_objects(compiler_output: str) -> list[dict]: + + return json_objects + + +def extract_warnings_from_compiler_output(compiler_output: str) -> list[dict]: + """ + Extracts warnings from the compiler output when using -fdiagnostics-format=json + + Compiler output as a whole is not a valid json document, but includes many json + objects and may include other output that is not json. + """ + # Extract JSON objects from the raw compiler output + compiler_output_json_objects = [] + stack = [] + start_index = None + for index, char in enumerate(compiler_output): + if char == '[': + if len(stack) == 0: + start_index = index # Start of a new JSON array + stack.append(char) + elif char == ']': + if len(stack) > 0: + stack.pop() + if len(stack) == 0 and start_index is not None: + try: + json_data = json.loads(compiler_output[start_index:index+1]) + compiler_output_json_objects.extend(json_data) + start_index = None + except json.JSONDecodeError: + continue # Skip malformed JSON + + compiler_warnings = [entry for entry in compiler_output_json_objects if entry.get('kind') == 'warning'] + + return compiler_warnings + + + +def get_unexpected_warnings( + warnings: list[dict], + files_with_expected_warnings: set[str], +) -> int: + """ + Fails if there are unexpected warnings + """ + unexpected_warnings = [] + for warning in warnings: + locations = warning['locations'] + for location in locations: + for key in ['caret', 'start', 'end']: + if key in location: + filename = location[key]['file'] + if filename not in files_with_expected_warnings: + unexpected_warnings.append(warning) + + if unexpected_warnings: + print("Unexpected warnings:") + for warning in unexpected_warnings: + print(warning) + return 1 + + + return 0 + + +def get_unexpected_improvements( + warnings: list[dict], + files_with_expected_warnings: set[str], +) -> int: + """ + Fails if files that were expected to have warnings have no warnings + """ + + # Create set of files with warnings + files_with_ewarnings = set() + for warning in warnings: + locations = warning['locations'] + for location in locations: + for key in ['caret', 'start', 'end']: + if key in location: + filename = location[key]['file'] + files_with_ewarnings.add(filename) + + + + unexpected_improvements = [] + for filename in files_with_expected_warnings: + if filename not in files_with_ewarnings: + unexpected_improvements.append(filename) + + if unexpected_improvements: + print("Unexpected improvements:") + for filename in unexpected_improvements: + print(filename) + return 1 + + + return 0 + + + + + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--compiler-output-file-path", + type=str, + required=True, + help="Path to the file" + ) + parser.add_argument( + "--warning-ignore-file-path", + type=str, + required=True, + help="Path to the warning ignore file" + ) + + + args = parser.parse_args(argv) + + exit_code = 0 + + # Check that the compiler output file is a valid path + if not Path(args.compiler_output_file_path).is_file(): + print(f"Compiler output file does not exist: {args.compiler_output_file_path}") + return 1 + + # Check that the warning ignore file is a valid path + if not Path(args.warning_ignore_file_path).is_file(): + print(f"Warning ignore file does not exist: {args.warning_ignore_file_path}") + return 1 + + with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f: + compiler_output_file_contents = f.read() + + with Path(args.warning_ignore_file_path).open(encoding="UTF-8") as clean_files: + files_with_expected_warnings = { + filename.strip() + for filename in clean_files + if filename.strip() and not filename.startswith("#") + } + + + if len(compiler_output_file_contents) > 0: + print("Successfully got compiler output") + else: + exit_code = 1 + + if len(files_with_expected_warnings) > 0: + print("we have some exceptions") + + + warnings = extract_warnings_from_compiler_output(compiler_output_file_contents) + + exit_code = get_unexpected_warnings(warnings, files_with_expected_warnings) + + exit_code = get_unexpected_improvements(warnings, files_with_expected_warnings) + + + + + return exit_code + + + + + + + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file From 3bd5a10a9b57d3eafb94bc4a9a739f5834094593 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Sat, 13 Jul 2024 13:18:25 -0500 Subject: [PATCH 002/115] Add flags to check warnings script to fail on regression or improvement --- Tools/build/check_warnings.py | 81 ++++++++++++++--------------------- 1 file changed, 33 insertions(+), 48 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index b2119b40f3de6e..daa60fb5282c9f 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -1,15 +1,13 @@ #!/usr/bin/env python3 +""" +Parses compiler output with -fdiagnostics-format=json and checks that warnings exist +only in files that are expected to have warnings. +""" import sys from pathlib import Path import argparse import json - -def extract_json_objects(compiler_output: str) -> list[dict]: - - return json_objects - - def extract_warnings_from_compiler_output(compiler_output: str) -> list[dict]: """ Extracts warnings from the compiler output when using -fdiagnostics-format=json @@ -41,14 +39,13 @@ def extract_warnings_from_compiler_output(compiler_output: str) -> list[dict]: return compiler_warnings - - def get_unexpected_warnings( warnings: list[dict], files_with_expected_warnings: set[str], ) -> int: """ - Fails if there are unexpected warnings + Returns failure status if warnings discovered in list of warnings are associated with a file + that is not found in the list of files with expected warnings """ unexpected_warnings = [] for warning in warnings: @@ -66,7 +63,6 @@ def get_unexpected_warnings( print(warning) return 1 - return 0 @@ -75,24 +71,23 @@ def get_unexpected_improvements( files_with_expected_warnings: set[str], ) -> int: """ - Fails if files that were expected to have warnings have no warnings + Returns failure status if there are no warnings in the list of warnings for a file + that is in the list of files with expected warnings """ # Create set of files with warnings - files_with_ewarnings = set() + files_with_warnings = set() for warning in warnings: locations = warning['locations'] for location in locations: for key in ['caret', 'start', 'end']: if key in location: filename = location[key]['file'] - files_with_ewarnings.add(filename) - - + files_with_warnings.add(filename) unexpected_improvements = [] for filename in files_with_expected_warnings: - if filename not in files_with_ewarnings: + if filename not in files_with_warnings: unexpected_improvements.append(filename) if unexpected_improvements: @@ -101,21 +96,15 @@ def get_unexpected_improvements( print(filename) return 1 - return 0 - - - - - def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( "--compiler-output-file-path", type=str, required=True, - help="Path to the file" + help="Path to the compiler output file" ) parser.add_argument( "--warning-ignore-file-path", @@ -123,7 +112,18 @@ def main(argv: list[str] | None = None) -> int: required=True, help="Path to the warning ignore file" ) - + parser.add_argument( + "--fail-on-regression", + action="store_true", + default=False, + help="Flag to fail if new warnings are found" + ) + parser.add_argument( + "--fail-on-improvement", + action="store_true", + default=False, + help="Flag to fail if files that were expected to have warnings have no warnings" + ) args = parser.parse_args(argv) @@ -133,12 +133,12 @@ def main(argv: list[str] | None = None) -> int: if not Path(args.compiler_output_file_path).is_file(): print(f"Compiler output file does not exist: {args.compiler_output_file_path}") return 1 - + # Check that the warning ignore file is a valid path if not Path(args.warning_ignore_file_path).is_file(): print(f"Warning ignore file does not exist: {args.warning_ignore_file_path}") return 1 - + with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f: compiler_output_file_contents = f.read() @@ -149,33 +149,18 @@ def main(argv: list[str] | None = None) -> int: if filename.strip() and not filename.startswith("#") } - - if len(compiler_output_file_contents) > 0: - print("Successfully got compiler output") - else: - exit_code = 1 - - if len(files_with_expected_warnings) > 0: - print("we have some exceptions") - + if not len(compiler_output_file_contents) > 0:exit_code = 1 warnings = extract_warnings_from_compiler_output(compiler_output_file_contents) - exit_code = get_unexpected_warnings(warnings, files_with_expected_warnings) - - exit_code = get_unexpected_improvements(warnings, files_with_expected_warnings) - - + status = get_unexpected_warnings(warnings, files_with_expected_warnings) + if args.fail_on_regression: exit_code |= status + status = get_unexpected_improvements(warnings, files_with_expected_warnings) + if args.fail_on_improvement: exit_code |= status + print("Returning exit code: ", exit_code) return exit_code - - - - - - - if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) From 68130035e6f1f9ca16af4defe38ff12d46f5eb28 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Sat, 13 Jul 2024 13:34:59 -0500 Subject: [PATCH 003/115] Remove redundant comment --- Tools/build/check_warnings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index daa60fb5282c9f..2e2c6b5b8a9d45 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -15,7 +15,6 @@ def extract_warnings_from_compiler_output(compiler_output: str) -> list[dict]: Compiler output as a whole is not a valid json document, but includes many json objects and may include other output that is not json. """ - # Extract JSON objects from the raw compiler output compiler_output_json_objects = [] stack = [] start_index = None From ab4d754775810ea8669868e00a30e5097e6c37ea Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Sat, 13 Jul 2024 14:24:34 -0500 Subject: [PATCH 004/115] Rename warnigore file to warningignore --- Tools/build/{.warnignore_ubuntu => .warningignore_ubuntu} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Tools/build/{.warnignore_ubuntu => .warningignore_ubuntu} (100%) diff --git a/Tools/build/.warnignore_ubuntu b/Tools/build/.warningignore_ubuntu similarity index 100% rename from Tools/build/.warnignore_ubuntu rename to Tools/build/.warningignore_ubuntu From 615d2282aa9baa5183acedb47bf950633b085486 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Sat, 13 Jul 2024 16:13:45 -0500 Subject: [PATCH 005/115] Use regex to extract json arrays --- Tools/build/check_warnings.py | 63 +++++++++++++++-------------------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 2e2c6b5b8a9d45..b655b11f707449 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -7,6 +7,7 @@ from pathlib import Path import argparse import json +import re def extract_warnings_from_compiler_output(compiler_output: str) -> list[dict]: """ @@ -15,26 +16,17 @@ def extract_warnings_from_compiler_output(compiler_output: str) -> list[dict]: Compiler output as a whole is not a valid json document, but includes many json objects and may include other output that is not json. """ - compiler_output_json_objects = [] - stack = [] - start_index = None - for index, char in enumerate(compiler_output): - if char == '[': - if len(stack) == 0: - start_index = index # Start of a new JSON array - stack.append(char) - elif char == ']': - if len(stack) > 0: - stack.pop() - if len(stack) == 0 and start_index is not None: - try: - json_data = json.loads(compiler_output[start_index:index+1]) - compiler_output_json_objects.extend(json_data) - start_index = None - except json.JSONDecodeError: - continue # Skip malformed JSON - - compiler_warnings = [entry for entry in compiler_output_json_objects if entry.get('kind') == 'warning'] + + # Regex to find json arrays at the top level of the file in the compiler output + json_arrays = re.findall(r'\[(?:[^\[\]]|\[(?:[^\[\]]|\[[^\[\]]*\])*\])*\]', compiler_output) + compiler_warnings = [] + for array in json_arrays: + try: + json_data = json.loads(array) + json_objects_in_array = [entry for entry in json_data] + compiler_warnings.extend([entry for entry in json_objects_in_array if entry.get('kind') == 'warning']) + except json.JSONDecodeError: + continue # Skip malformed JSON return compiler_warnings @@ -52,8 +44,9 @@ def get_unexpected_warnings( for location in locations: for key in ['caret', 'start', 'end']: if key in location: - filename = location[key]['file'] - if filename not in files_with_expected_warnings: + file = location[key]['file'] + file = file.lstrip('./') # Remove leading curdir if present + if file not in files_with_expected_warnings: unexpected_warnings.append(warning) if unexpected_warnings: @@ -81,18 +74,19 @@ def get_unexpected_improvements( for location in locations: for key in ['caret', 'start', 'end']: if key in location: - filename = location[key]['file'] - files_with_warnings.add(filename) + file = location[key]['file'] + file = file.lstrip('./') # Remove leading curdir if present + files_with_warnings.add(file) unexpected_improvements = [] - for filename in files_with_expected_warnings: - if filename not in files_with_warnings: - unexpected_improvements.append(filename) + for file in files_with_expected_warnings: + if file not in files_with_warnings: + unexpected_improvements.append(file) if unexpected_improvements: print("Unexpected improvements:") - for filename in unexpected_improvements: - print(filename) + for file in unexpected_improvements: + print(file) return 1 return 0 @@ -143,13 +137,11 @@ def main(argv: list[str] | None = None) -> int: with Path(args.warning_ignore_file_path).open(encoding="UTF-8") as clean_files: files_with_expected_warnings = { - filename.strip() - for filename in clean_files - if filename.strip() and not filename.startswith("#") + file.strip() + for file in clean_files + if file.strip() and not file.startswith("#") } - if not len(compiler_output_file_contents) > 0:exit_code = 1 - warnings = extract_warnings_from_compiler_output(compiler_output_file_contents) status = get_unexpected_warnings(warnings, files_with_expected_warnings) @@ -157,8 +149,7 @@ def main(argv: list[str] | None = None) -> int: status = get_unexpected_improvements(warnings, files_with_expected_warnings) if args.fail_on_improvement: exit_code |= status - - print("Returning exit code: ", exit_code) + return exit_code if __name__ == "__main__": From fc0a60ba7e6a902b0abd17faa30b18c97f1ad90c Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Sat, 13 Jul 2024 16:16:29 -0500 Subject: [PATCH 006/115] Trim whitespace --- Tools/build/check_warnings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index b655b11f707449..16075d855a6e39 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -149,7 +149,7 @@ def main(argv: list[str] | None = None) -> int: status = get_unexpected_improvements(warnings, files_with_expected_warnings) if args.fail_on_improvement: exit_code |= status - + return exit_code if __name__ == "__main__": From 7793d80836c9fa1adc7841acf25f4f135fdeaa99 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Sat, 13 Jul 2024 16:30:09 -0500 Subject: [PATCH 007/115] Test on github unexpected improvement --- Tools/build/.warningignore_ubuntu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 8242c8d17c89fb..83dccf0eb215f6 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -1,3 +1,5 @@ # Files listed will be ignored by the compiler warning checker # for the Ubuntu/build and test job. # Keep lines sorted lexicographically to help avoid merge conflicts. + +Modules/_decimal/libmpdec/memory.c \ No newline at end of file From 99715d235cc437e6b5ce7cb194e34559149296c2 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Sat, 13 Jul 2024 16:36:42 -0500 Subject: [PATCH 008/115] Add config for improve fail check --- .github/workflows/reusable-ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 3a6f2b910d0da0..9b05e1141d357c 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -79,7 +79,7 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo - name: Check compiler warnings - run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu + run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --fail-on-improvement - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw From bc44ec223af3ed552b0d769ffd5629f8a540ebf2 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Sat, 13 Jul 2024 16:38:27 -0500 Subject: [PATCH 009/115] Revert to prod check warning state --- .github/workflows/reusable-ubuntu.yml | 2 +- Tools/build/.warningignore_ubuntu | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 9b05e1141d357c..3a6f2b910d0da0 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -79,7 +79,7 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo - name: Check compiler warnings - run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --fail-on-improvement + run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 83dccf0eb215f6..8242c8d17c89fb 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -1,5 +1,3 @@ # Files listed will be ignored by the compiler warning checker # for the Ubuntu/build and test job. # Keep lines sorted lexicographically to help avoid merge conflicts. - -Modules/_decimal/libmpdec/memory.c \ No newline at end of file From e2ca75f3991e20b453622b9917fc6182938ed57b Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 21:55:58 +0000 Subject: [PATCH 010/115] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by?= =?UTF-8?q?=20blurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst diff --git a/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst b/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst new file mode 100644 index 00000000000000..27ccd8e63d5d9e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst @@ -0,0 +1 @@ +Add tooling to check for changes in compiler warnings. From bd1634e3c3504b4b79423c9f65a50bbf30205241 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 16 Jul 2024 22:32:12 -0500 Subject: [PATCH 011/115] Refactor creating set of files with warnings to a dedicated function --- ...-07-13-21-55-58.gh-issue-112301.YJS1dl.rst | 1 + Tools/build/check_warnings.py | 61 ++++++++++--------- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst b/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst index 27ccd8e63d5d9e..4dd10fb746d44d 100644 --- a/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst +++ b/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst @@ -1 +1,2 @@ Add tooling to check for changes in compiler warnings. +Patch by Nate Ohlson \ No newline at end of file diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 16075d855a6e39..6c048ecd845dfa 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -3,11 +3,11 @@ Parses compiler output with -fdiagnostics-format=json and checks that warnings exist only in files that are expected to have warnings. """ -import sys -from pathlib import Path import argparse import json import re +import sys +from pathlib import Path def extract_warnings_from_compiler_output(compiler_output: str) -> list[dict]: """ @@ -24,63 +24,67 @@ def extract_warnings_from_compiler_output(compiler_output: str) -> list[dict]: try: json_data = json.loads(array) json_objects_in_array = [entry for entry in json_data] - compiler_warnings.extend([entry for entry in json_objects_in_array if entry.get('kind') == 'warning']) + compiler_warnings.extend([entry for entry in json_objects_in_array + if entry.get('kind') == 'warning']) except json.JSONDecodeError: continue # Skip malformed JSON return compiler_warnings +# Create a function that returns a dictionary where the key is the file and the data is the warnings in that file +def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: + """ + Returns a dictionary where the key is the file and the data is the warnings in that file + """ + warnings_by_file = {} + for warning in warnings: + locations = warning['locations'] + for location in locations: + for key in ['caret', 'start', 'end']: + if key in location: + file = location[key]['file'] + file = file.lstrip('./') # Remove leading current directory if present + if file not in warnings_by_file: + warnings_by_file[file] = [] + warnings_by_file[file].append(warning) + + return warnings_by_file + def get_unexpected_warnings( warnings: list[dict], files_with_expected_warnings: set[str], + files_with_warnings: set[str], ) -> int: """ Returns failure status if warnings discovered in list of warnings are associated with a file that is not found in the list of files with expected warnings """ unexpected_warnings = [] - for warning in warnings: - locations = warning['locations'] - for location in locations: - for key in ['caret', 'start', 'end']: - if key in location: - file = location[key]['file'] - file = file.lstrip('./') # Remove leading curdir if present - if file not in files_with_expected_warnings: - unexpected_warnings.append(warning) + for file in files_with_warnings.keys(): + if file not in files_with_expected_warnings: + unexpected_warnings.extend(files_with_warnings[file]) if unexpected_warnings: print("Unexpected warnings:") for warning in unexpected_warnings: print(warning) return 1 - + return 0 def get_unexpected_improvements( warnings: list[dict], files_with_expected_warnings: set[str], + files_with_warnings: set[str], ) -> int: """ Returns failure status if there are no warnings in the list of warnings for a file that is in the list of files with expected warnings """ - - # Create set of files with warnings - files_with_warnings = set() - for warning in warnings: - locations = warning['locations'] - for location in locations: - for key in ['caret', 'start', 'end']: - if key in location: - file = location[key]['file'] - file = file.lstrip('./') # Remove leading curdir if present - files_with_warnings.add(file) - unexpected_improvements = [] for file in files_with_expected_warnings: - if file not in files_with_warnings: + if file not in files_with_warnings.keys(): unexpected_improvements.append(file) if unexpected_improvements: @@ -143,11 +147,12 @@ def main(argv: list[str] | None = None) -> int: } warnings = extract_warnings_from_compiler_output(compiler_output_file_contents) + files_with_warnings = get_warnings_by_file(warnings) - status = get_unexpected_warnings(warnings, files_with_expected_warnings) + status = get_unexpected_warnings(warnings, files_with_expected_warnings, files_with_warnings) if args.fail_on_regression: exit_code |= status - status = get_unexpected_improvements(warnings, files_with_expected_warnings) + status = get_unexpected_improvements(warnings, files_with_expected_warnings, files_with_warnings) if args.fail_on_improvement: exit_code |= status return exit_code From b07b1d6241840639a5c2d90a31644e02d245b273 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 23 Jul 2024 02:41:34 -0500 Subject: [PATCH 012/115] Move cflags configure option to top level build configuration --- Tools/build/check_warnings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 6c048ecd845dfa..1330a14f91ba5d 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -69,9 +69,8 @@ def get_unexpected_warnings( for warning in unexpected_warnings: print(warning) return 1 - - return 0 + return 0 def get_unexpected_improvements( warnings: list[dict], From e1954a53dd2b25098ebeb5b301cdb3188a61a5aa Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 23 Jul 2024 03:20:24 -0500 Subject: [PATCH 013/115] Add json diagnostics to ubuntu configuration as first argument --- .github/workflows/reusable-ubuntu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 3a6f2b910d0da0..c6289a74e9a5f6 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -67,6 +67,7 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: >- ../cpython-ro-srcdir/configure + CFLAGS="-fdiagnostics-format=json" --config-cache --with-pydebug --enable-slower-safety From 59351430878fb1195d5a1d309f2ae6f16c384c91 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 23 Jul 2024 15:44:55 -0500 Subject: [PATCH 014/115] Add newline to news --- .../next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst b/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst index 4dd10fb746d44d..4e8a8b6dd36ba6 100644 --- a/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst +++ b/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst @@ -1,2 +1,2 @@ Add tooling to check for changes in compiler warnings. -Patch by Nate Ohlson \ No newline at end of file +Patch by Nate Ohlson From 7f1a238b3a26fc8fcee4629dcb62753f369b7fa2 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Sat, 13 Jul 2024 12:50:17 -0500 Subject: [PATCH 015/115] Create simple warning check tool and add to ubuntu build and test job --- Tools/build/.warnignore_ubuntu | 3 +++ Tools/build/check_warnings.py | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 Tools/build/.warnignore_ubuntu diff --git a/Tools/build/.warnignore_ubuntu b/Tools/build/.warnignore_ubuntu new file mode 100644 index 00000000000000..8242c8d17c89fb --- /dev/null +++ b/Tools/build/.warnignore_ubuntu @@ -0,0 +1,3 @@ +# Files listed will be ignored by the compiler warning checker +# for the Ubuntu/build and test job. +# Keep lines sorted lexicographically to help avoid merge conflicts. diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 1330a14f91ba5d..712a505d057f4c 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -129,12 +129,10 @@ def main(argv: list[str] | None = None) -> int: if not Path(args.compiler_output_file_path).is_file(): print(f"Compiler output file does not exist: {args.compiler_output_file_path}") return 1 - # Check that the warning ignore file is a valid path if not Path(args.warning_ignore_file_path).is_file(): print(f"Warning ignore file does not exist: {args.warning_ignore_file_path}") return 1 - with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f: compiler_output_file_contents = f.read() From 144136e8c7b3a4a8b846954e20f1be038bfbb52a Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 22 Jul 2024 13:59:08 -0500 Subject: [PATCH 016/115] Add macos warning checks to GitHub actions --- .github/workflows/reusable-macos.yml | 10 +++++++--- Tools/build/.warningignore_macos | 3 +++ 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 Tools/build/.warningignore_macos diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 0f189960dbea61..16e7bc71ff5dfd 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -26,13 +26,15 @@ jobs: strategy: fail-fast: false matrix: - os: ${{fromJson(inputs.os-matrix)}} + os: + - "macos-10.15" + - "macos-11" is-fork: - ${{ github.repository_owner != 'python' }} exclude: - os: "ghcr.io/cirruslabs/macos-runner:sonoma" is-fork: true - - os: "macos-14" + - os: "macos-10.15" is-fork: false runs-on: ${{ matrix.os }} steps: @@ -58,8 +60,10 @@ jobs: --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - run: make -j8 + run: make -j8 &> compiler_output.txt - name: Display build info run: make pythoninfo + - name: Check compiler warnings + run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_macos - name: Tests run: make test diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos new file mode 100644 index 00000000000000..1b504dfc54000f --- /dev/null +++ b/Tools/build/.warningignore_macos @@ -0,0 +1,3 @@ +# Files listed will be ignored by the compiler warning checker +# for the macOS/build and test job. +# Keep lines sorted lexicographically to help avoid merge conflicts. From 3dd40fd960d086ad41f57d1a03d5e83f63095dd6 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 22 Jul 2024 14:32:34 -0500 Subject: [PATCH 017/115] Revert reusable-macos.yml for environment variables --- .github/workflows/reusable-macos.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 16e7bc71ff5dfd..e54de82a9ecf4a 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -26,15 +26,13 @@ jobs: strategy: fail-fast: false matrix: - os: - - "macos-10.15" - - "macos-11" + os: ${{fromJson(inputs.os-matrix)}} is-fork: - ${{ github.repository_owner != 'python' }} exclude: - os: "ghcr.io/cirruslabs/macos-runner:sonoma" is-fork: true - - os: "macos-10.15" + - os: "macos-14" is-fork: false runs-on: ${{ matrix.os }} steps: From 1158f53bc5e4976fc18fef9b460ede66c2bca3d2 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 22 Jul 2024 14:46:16 -0500 Subject: [PATCH 018/115] Update paths --- .github/workflows/reusable-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index e54de82a9ecf4a..0751f2f63a564b 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -62,6 +62,6 @@ jobs: - name: Display build info run: make pythoninfo - name: Check compiler warnings - run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_macos + run: python Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos - name: Tests run: make test From cb51b4f854cf2c8c65c5f65b6c24d89b3c17abeb Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 22 Jul 2024 15:03:30 -0500 Subject: [PATCH 019/115] Test unexpected improvement --- Tools/build/.warningignore_macos | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos index 1b504dfc54000f..ec6dc2c54367e9 100644 --- a/Tools/build/.warningignore_macos +++ b/Tools/build/.warningignore_macos @@ -1,3 +1,5 @@ # Files listed will be ignored by the compiler warning checker # for the macOS/build and test job. # Keep lines sorted lexicographically to help avoid merge conflicts. + +Modules/_abc.c \ No newline at end of file From 77e0f6e7850d3696fffaa4c08e73053a9f8922ef Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 22 Jul 2024 16:08:53 -0500 Subject: [PATCH 020/115] Remove warning ignore --- Tools/build/.warningignore_macos | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos index ec6dc2c54367e9..1b504dfc54000f 100644 --- a/Tools/build/.warningignore_macos +++ b/Tools/build/.warningignore_macos @@ -1,5 +1,3 @@ # Files listed will be ignored by the compiler warning checker # for the macOS/build and test job. # Keep lines sorted lexicographically to help avoid merge conflicts. - -Modules/_abc.c \ No newline at end of file From 3e1d75f14c87505055e4b5a9c269068253a29b76 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 22 Jul 2024 16:57:19 -0500 Subject: [PATCH 021/115] Add json output option to macos configure job --- .github/workflows/reusable-macos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 0751f2f63a564b..6241b11fa6106f 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -51,6 +51,7 @@ jobs: GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ ./configure \ + CFLAGS="--fdiagnostics-format=json" \ --config-cache \ --with-pydebug \ --enable-slower-safety \ From b5cd58ae585710776f2d48410f2146e3a45fc065 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 23 Jul 2024 00:09:14 -0500 Subject: [PATCH 022/115] Add common dictionary format when parsing warnings --- .github/workflows/reusable-macos.yml | 2 +- .github/workflows/reusable-ubuntu.yml | 2 +- Tools/build/check_warnings.py | 70 ++++++++++++++++++++------- 3 files changed, 55 insertions(+), 19 deletions(-) diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 6241b11fa6106f..9adda38cf165d2 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -63,6 +63,6 @@ jobs: - name: Display build info run: make pythoninfo - name: Check compiler warnings - run: python Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos + run: python Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang - name: Tests run: make test diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index c6289a74e9a5f6..670dec62c4ea30 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -80,7 +80,7 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo - name: Check compiler warnings - run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu + run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 712a505d057f4c..5c359a69462c9f 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -9,7 +9,27 @@ import sys from pathlib import Path -def extract_warnings_from_compiler_output(compiler_output: str) -> list[dict]: +def extract_warnings_from_compiler_output_clang(compiler_output: str) -> list[dict]: + """ + Extracts warnings from the compiler output when using clang + """ + + # Regex to find warnings in the compiler output + clang_warning_regex = re.compile(r'(?P.*):(?P\d+):(?P\d+): warning: (?P.*)') + compiler_warnings = [] + for line in compiler_output.splitlines(): + match = clang_warning_regex.match(line) + if match: + compiler_warnings.append({ + 'file': match.group('file'), + 'line': match.group('line'), + 'column': match.group('column'), + 'message': match.group('message'), + }) + + return compiler_warnings + +def extract_warnings_from_compiler_output_json(compiler_output: str) -> list[dict]: """ Extracts warnings from the compiler output when using -fdiagnostics-format=json @@ -24,8 +44,20 @@ def extract_warnings_from_compiler_output(compiler_output: str) -> list[dict]: try: json_data = json.loads(array) json_objects_in_array = [entry for entry in json_data] - compiler_warnings.extend([entry for entry in json_objects_in_array - if entry.get('kind') == 'warning']) + warning_list = [entry for entry in json_objects_in_array if entry.get('kind') == 'warning'] + for warning in warning_list: + locations = warning['locations'] + for location in locations: + for key in ['caret', 'start', 'end']: + if key in location: + compiler_warnings.append({ + 'file': location[key]['file'].lstrip('./'), # Remove leading current directory if present + 'line': location[key]['line'], + 'column': location[key]['column'], + 'message': warning['message'], + }) + + except json.JSONDecodeError: continue # Skip malformed JSON @@ -38,20 +70,14 @@ def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: """ warnings_by_file = {} for warning in warnings: - locations = warning['locations'] - for location in locations: - for key in ['caret', 'start', 'end']: - if key in location: - file = location[key]['file'] - file = file.lstrip('./') # Remove leading current directory if present - if file not in warnings_by_file: - warnings_by_file[file] = [] - warnings_by_file[file].append(warning) + file = warning['file'] + if file not in warnings_by_file: + warnings_by_file[file] = [] + warnings_by_file[file].append(warning) return warnings_by_file def get_unexpected_warnings( - warnings: list[dict], files_with_expected_warnings: set[str], files_with_warnings: set[str], ) -> int: @@ -73,7 +99,6 @@ def get_unexpected_warnings( return 0 def get_unexpected_improvements( - warnings: list[dict], files_with_expected_warnings: set[str], files_with_warnings: set[str], ) -> int: @@ -120,6 +145,13 @@ def main(argv: list[str] | None = None) -> int: default=False, help="Flag to fail if files that were expected to have warnings have no warnings" ) + parser.add_argument( + "--compiler-output-type", + type=str, + required=True, + choices=["json", "clang"], + help="Type of compiler output file (json or clang)" + ) args = parser.parse_args(argv) @@ -143,13 +175,17 @@ def main(argv: list[str] | None = None) -> int: if file.strip() and not file.startswith("#") } - warnings = extract_warnings_from_compiler_output(compiler_output_file_contents) + if args.compiler_output_type == "json": + warnings = extract_warnings_from_compiler_output_json(compiler_output_file_contents) + elif args.compiler_output_type == "clang": + warnings = extract_warnings_from_compiler_output_clang(compiler_output_file_contents) + files_with_warnings = get_warnings_by_file(warnings) - status = get_unexpected_warnings(warnings, files_with_expected_warnings, files_with_warnings) + status = get_unexpected_warnings(files_with_expected_warnings, files_with_warnings) if args.fail_on_regression: exit_code |= status - status = get_unexpected_improvements(warnings, files_with_expected_warnings, files_with_warnings) + status = get_unexpected_improvements(files_with_expected_warnings, files_with_warnings) if args.fail_on_improvement: exit_code |= status return exit_code From 02f313e37bbf052fa1e7a09cb2c29d38abf9d095 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 23 Jul 2024 00:11:42 -0500 Subject: [PATCH 023/115] Remove configure option for macos job --- .github/workflows/reusable-macos.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 9adda38cf165d2..530855212ba33e 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -51,7 +51,6 @@ jobs: GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ ./configure \ - CFLAGS="--fdiagnostics-format=json" \ --config-cache \ --with-pydebug \ --enable-slower-safety \ From 83d1ed7100f45d0d43bf823dc9087107b93a8272 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 23 Jul 2024 01:55:22 -0500 Subject: [PATCH 024/115] Print out json version of compiler output --- Tools/build/check_warnings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 5c359a69462c9f..c6061085a12bd3 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -36,6 +36,9 @@ def extract_warnings_from_compiler_output_json(compiler_output: str) -> list[dic Compiler output as a whole is not a valid json document, but includes many json objects and may include other output that is not json. """ + print("######") + print(compiler_output) + print("&&&&&") # Regex to find json arrays at the top level of the file in the compiler output json_arrays = re.findall(r'\[(?:[^\[\]]|\[(?:[^\[\]]|\[[^\[\]]*\])*\])*\]', compiler_output) From 08a6f6d1d06ecf81fae7258adfebee164ae1f0a5 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 23 Jul 2024 16:22:50 -0500 Subject: [PATCH 025/115] Remove old version of warning ignore file --- Tools/build/.warnignore_ubuntu | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 Tools/build/.warnignore_ubuntu diff --git a/Tools/build/.warnignore_ubuntu b/Tools/build/.warnignore_ubuntu deleted file mode 100644 index 8242c8d17c89fb..00000000000000 --- a/Tools/build/.warnignore_ubuntu +++ /dev/null @@ -1,3 +0,0 @@ -# Files listed will be ignored by the compiler warning checker -# for the Ubuntu/build and test job. -# Keep lines sorted lexicographically to help avoid merge conflicts. From 7be8ee6b1ea327c161f7d3f268ac169d726add1b Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Wed, 24 Jul 2024 00:15:26 -0500 Subject: [PATCH 026/115] Remove compiler output print diagnostic --- Tools/build/check_warnings.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index c6061085a12bd3..f5e8ec8efdd943 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -13,7 +13,6 @@ def extract_warnings_from_compiler_output_clang(compiler_output: str) -> list[di """ Extracts warnings from the compiler output when using clang """ - # Regex to find warnings in the compiler output clang_warning_regex = re.compile(r'(?P.*):(?P\d+):(?P\d+): warning: (?P.*)') compiler_warnings = [] @@ -36,10 +35,6 @@ def extract_warnings_from_compiler_output_json(compiler_output: str) -> list[dic Compiler output as a whole is not a valid json document, but includes many json objects and may include other output that is not json. """ - print("######") - print(compiler_output) - print("&&&&&") - # Regex to find json arrays at the top level of the file in the compiler output json_arrays = re.findall(r'\[(?:[^\[\]]|\[(?:[^\[\]]|\[[^\[\]]*\])*\])*\]', compiler_output) compiler_warnings = [] From 8b0a2eeaec843dbb5ba5defcf43acd37cf28a0fa Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 24 Jul 2024 05:18:26 +0000 Subject: [PATCH 027/115] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by?= =?UTF-8?q?=20blurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Security/2024-07-24-05-18-25.gh-issue-112301.lfINgZ.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2024-07-24-05-18-25.gh-issue-112301.lfINgZ.rst diff --git a/Misc/NEWS.d/next/Security/2024-07-24-05-18-25.gh-issue-112301.lfINgZ.rst b/Misc/NEWS.d/next/Security/2024-07-24-05-18-25.gh-issue-112301.lfINgZ.rst new file mode 100644 index 00000000000000..bc9f6758108ce1 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-07-24-05-18-25.gh-issue-112301.lfINgZ.rst @@ -0,0 +1,2 @@ +Add macOS warning tracking to warning check tooling. +Patch by Nate Ohlson From 49cbd87cc6c4c88bf5721c601e36e50bbb51cbae Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Wed, 24 Jul 2024 00:21:37 -0500 Subject: [PATCH 028/115] Remove superfluous comment --- Tools/build/check_warnings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index f5e8ec8efdd943..9771977d6e705c 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -61,7 +61,6 @@ def extract_warnings_from_compiler_output_json(compiler_output: str) -> list[dic return compiler_warnings -# Create a function that returns a dictionary where the key is the file and the data is the warnings in that file def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: """ Returns a dictionary where the key is the file and the data is the warnings in that file From b654a84c46debebe2edbb63f19d646a8ff0928d8 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 30 Jul 2024 10:26:34 -0500 Subject: [PATCH 029/115] Add period to news --- .../Security/2024-07-24-05-18-25.gh-issue-112301.lfINgZ.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Security/2024-07-24-05-18-25.gh-issue-112301.lfINgZ.rst b/Misc/NEWS.d/next/Security/2024-07-24-05-18-25.gh-issue-112301.lfINgZ.rst index bc9f6758108ce1..81237e735ebdb7 100644 --- a/Misc/NEWS.d/next/Security/2024-07-24-05-18-25.gh-issue-112301.lfINgZ.rst +++ b/Misc/NEWS.d/next/Security/2024-07-24-05-18-25.gh-issue-112301.lfINgZ.rst @@ -1,2 +1,2 @@ -Add macOS warning tracking to warning check tooling. -Patch by Nate Ohlson +Add macOS warning tracking to warning check tooling. +Patch by Nate Ohlson. From 3688c5c2639d22b28c15a74b83ce912a775c2201 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 30 Jul 2024 22:20:43 -0500 Subject: [PATCH 030/115] Make warning ignore file optional --- Tools/build/check_warnings.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 77347c752ef6e9..e936bc372f5cc1 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -151,7 +151,6 @@ def main(argv: list[str] | None = None) -> int: "-i", "--warning-ignore-file-path", type=str, - required=True, help="Path to the warning ignore file", ) parser.add_argument( @@ -187,24 +186,31 @@ def main(argv: list[str] | None = None) -> int: f"Compiler output file does not exist: {args.compiler_output_file_path}" ) return 1 - # Check that the warning ignore file is a valid path - if not Path(args.warning_ignore_file_path).is_file(): + + # Check that a warning ignore file was specified and if so is a valid path + if not args.warning_ignore_file_path: print( - f"Warning ignore file does not exist: {args.warning_ignore_file_path}" + "Warning ignore file not specified. Continuing without it (no warnings ignored)." ) - return 1 + files_with_expected_warnings = set() + else: + if not Path(args.warning_ignore_file_path).is_file(): + print( + f"Warning ignore file does not exist: {args.warning_ignore_file_path}" + ) + return 1 + with Path(args.warning_ignore_file_path).open( + encoding="UTF-8" + ) as clean_files: + files_with_expected_warnings = { + file.strip() + for file in clean_files + if file.strip() and not file.startswith("#") + } + with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f: compiler_output_file_contents = f.read() - with Path(args.warning_ignore_file_path).open( - encoding="UTF-8" - ) as clean_files: - files_with_expected_warnings = { - file.strip() - for file in clean_files - if file.strip() and not file.startswith("#") - } - if args.compiler_output_type == "json": warnings = extract_warnings_from_compiler_output_json( compiler_output_file_contents From cb1f276f5e30a8f05a4199a82afa64a8e56e2394 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 30 Jul 2024 22:49:35 -0500 Subject: [PATCH 031/115] Add write compiler output to log and file --- .github/workflows/reusable-macos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 8e3f7e52615478..d77723ef27c2dc 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -48,10 +48,10 @@ jobs: --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - run: make -j8 &> compiler_output.txt + run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt - name: Display build info run: make pythoninfo - name: Check compiler warnings - run: python Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang + run: python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang - name: Tests run: make test From fb91e3e2d38aa89b2779e18f94645733915b01f1 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Wed, 31 Jul 2024 12:02:40 -0500 Subject: [PATCH 032/115] Fix formatting and update regex --- Tools/build/check_warnings.py | 47 +++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index e936bc372f5cc1..ddadde6583264b 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -2,7 +2,9 @@ Parses compiler output with -fdiagnostics-format=json and checks that warnings exist only in files that are expected to have warnings. """ + import argparse +from collections import defaultdict import json import re import sys @@ -21,8 +23,7 @@ def extract_warnings_from_compiler_output_clang( ) compiler_warnings = [] for line in compiler_output.splitlines(): - match = clang_warning_regex.match(line) - if match: + if match := clang_warning_regex.match(line): compiler_warnings.append( { "file": match.group("file"), @@ -39,14 +40,16 @@ def extract_warnings_from_compiler_output_json( compiler_output: str, ) -> list[dict]: """ - Extracts warnings from the compiler output when using -fdiagnostics-format=json + Extracts warnings from the compiler output when using + -fdiagnostics-format=json. - Compiler output as a whole is not a valid json document, but includes many json - objects and may include other output that is not json. + Compiler output as a whole is not a valid json document, + but includes many json objects and may include other output + that is not json. """ # Regex to find json arrays at the top level of the file in the compiler output json_arrays = re.findall( - r"\[(?:[^\[\]]|\[(?:[^\[\]]|\[[^\[\]]*\])*\])*\]", compiler_output + r"\[(?:[^[\]]|\[[^\]]*\])*\]", compiler_output ) compiler_warnings = [] for array in json_arrays: @@ -61,18 +64,24 @@ def extract_warnings_from_compiler_output_json( for warning in warning_list: locations = warning["locations"] for location in locations: + found_location = False for key in ["caret", "start", "end"]: if key in location: compiler_warnings.append( { - "file": location[key]["file"].lstrip( - "./" - ), # Remove leading current directory if present + # Remove leading current directory if present + "file": location[key]["file"].lstrip("./"), "line": location[key]["line"], "column": location[key]["column"], "message": warning["message"], } ) + # Found a caret, start, or end in location so + # break out completely to address next warning + break + else: + continue + break except json.JSONDecodeError: continue # Skip malformed JSON @@ -84,19 +93,16 @@ def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: """ Returns a dictionary where the key is the file and the data is the warnings in that file """ - warnings_by_file = {} + warnings_by_file = defaultdict(list) for warning in warnings: - file = warning["file"] - if file not in warnings_by_file: - warnings_by_file[file] = [] - warnings_by_file[file].append(warning) + warnings_by_file[warning["file"]].append(warning) return warnings_by_file def get_unexpected_warnings( files_with_expected_warnings: set[str], - files_with_warnings: set[str], + files_with_warnings: dict[str, list[dict]], ) -> int: """ Returns failure status if warnings discovered in list of warnings are associated with a file @@ -118,7 +124,7 @@ def get_unexpected_warnings( def get_unexpected_improvements( files_with_expected_warnings: set[str], - files_with_warnings: set[str], + files_with_warnings: dict[str, list[dict]], ) -> int: """ Returns failure status if there are no warnings in the list of warnings for a file @@ -183,20 +189,23 @@ def main(argv: list[str] | None = None) -> int: # Check that the compiler output file is a valid path if not Path(args.compiler_output_file_path).is_file(): print( - f"Compiler output file does not exist: {args.compiler_output_file_path}" + f"Compiler output file does not exist:" + f" {args.compiler_output_file_path}" ) return 1 # Check that a warning ignore file was specified and if so is a valid path if not args.warning_ignore_file_path: print( - "Warning ignore file not specified. Continuing without it (no warnings ignored)." + "Warning ignore file not specified." + " Continuing without it (no warnings ignored)." ) files_with_expected_warnings = set() else: if not Path(args.warning_ignore_file_path).is_file(): print( - f"Warning ignore file does not exist: {args.warning_ignore_file_path}" + f"Warning ignore file does not exist:" + f" {args.warning_ignore_file_path}" ) return 1 with Path(args.warning_ignore_file_path).open( From f35ba6013af9db280073b0f23f5449f922f5c00a Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 5 Aug 2024 15:00:54 -0500 Subject: [PATCH 033/115] Fix comment formatting --- Tools/build/check_warnings.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index ddadde6583264b..31258932dbd4ca 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -47,7 +47,8 @@ def extract_warnings_from_compiler_output_json( but includes many json objects and may include other output that is not json. """ - # Regex to find json arrays at the top level of the file in the compiler output + # Regex to find json arrays at the top level of the file + # in the compiler output json_arrays = re.findall( r"\[(?:[^[\]]|\[[^\]]*\])*\]", compiler_output ) @@ -64,7 +65,6 @@ def extract_warnings_from_compiler_output_json( for warning in warning_list: locations = warning["locations"] for location in locations: - found_location = False for key in ["caret", "start", "end"]: if key in location: compiler_warnings.append( @@ -91,7 +91,8 @@ def extract_warnings_from_compiler_output_json( def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: """ - Returns a dictionary where the key is the file and the data is the warnings in that file + Returns a dictionary where the key is the file and the data is the warnings + in that file """ warnings_by_file = defaultdict(list) for warning in warnings: @@ -105,8 +106,9 @@ def get_unexpected_warnings( files_with_warnings: dict[str, list[dict]], ) -> int: """ - Returns failure status if warnings discovered in list of warnings are associated with a file - that is not found in the list of files with expected warnings + Returns failure status if warnings discovered in list of warnings + are associated with a file that is not found in the list of files + with expected warnings """ unexpected_warnings = [] for file in files_with_warnings.keys(): @@ -127,8 +129,8 @@ def get_unexpected_improvements( files_with_warnings: dict[str, list[dict]], ) -> int: """ - Returns failure status if there are no warnings in the list of warnings for a file - that is in the list of files with expected warnings + Returns failure status if there are no warnings in the list of warnings + for a file that is in the list of files with expected warnings """ unexpected_improvements = [] for file in files_with_expected_warnings: @@ -171,7 +173,8 @@ def main(argv: list[str] | None = None) -> int: "--fail-on-improvement", action="store_true", default=False, - help="Flag to fail if files that were expected to have warnings have no warnings", + help="Flag to fail if files that were expected " + "to have warnings have no warnings", ) parser.add_argument( "-t", From 294dd6163472312ecd0fb4c851cc48f434f99248 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 5 Aug 2024 18:20:24 -0500 Subject: [PATCH 034/115] Add warning count to check warnigns tooling --- Tools/build/.warningignore_macos | 2 ++ Tools/build/.warningignore_ubuntu | 2 ++ Tools/build/check_warnings.py | 18 ++++++++++++++---- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos index 1b504dfc54000f..67f50119db7310 100644 --- a/Tools/build/.warningignore_macos +++ b/Tools/build/.warningignore_macos @@ -1,3 +1,5 @@ # Files listed will be ignored by the compiler warning checker # for the macOS/build and test job. # Keep lines sorted lexicographically to help avoid merge conflicts. +# Format example: +# /path/to/file (number of warnings in file) diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 8242c8d17c89fb..469c727abfb11c 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -1,3 +1,5 @@ # Files listed will be ignored by the compiler warning checker # for the Ubuntu/build and test job. # Keep lines sorted lexicographically to help avoid merge conflicts. +# Format example: +# /path/to/file (number of warnings in file) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 31258932dbd4ca..7a0932dc017c53 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -74,6 +74,7 @@ def extract_warnings_from_compiler_output_json( "line": location[key]["line"], "column": location[key]["column"], "message": warning["message"], + "option": warning["option"], } ) # Found a caret, start, or end in location so @@ -112,6 +113,11 @@ def get_unexpected_warnings( """ unexpected_warnings = [] for file in files_with_warnings.keys(): + for ignore_file in files_with_expected_warnings: + if file == ignore_file[0]: + if len(files_with_warnings[file]) > ignore_file[1]: + unexpected_warnings.extend(files_with_warnings[file]) + break if file not in files_with_expected_warnings: unexpected_warnings.extend(files_with_warnings[file]) @@ -134,8 +140,11 @@ def get_unexpected_improvements( """ unexpected_improvements = [] for file in files_with_expected_warnings: - if file not in files_with_warnings.keys(): + if file[0] not in files_with_warnings.keys(): unexpected_improvements.append(file) + else: + if (len(files_with_warnings[file[0]]) < file[1]): + unexpected_improvements.append(file) if unexpected_improvements: print("Unexpected improvements:") @@ -214,11 +223,12 @@ def main(argv: list[str] | None = None) -> int: with Path(args.warning_ignore_file_path).open( encoding="UTF-8" ) as clean_files: - files_with_expected_warnings = { - file.strip() + files_with_expected_warnings = [ + (file.strip().split()[0], int(file.strip().split()[1])) for file in clean_files if file.strip() and not file.startswith("#") - } + ] + with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f: compiler_output_file_contents = f.read() From 405698517dbc4491300b33562ae4d4af29e469f6 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 5 Aug 2024 18:53:52 -0500 Subject: [PATCH 035/115] Remove duplicate warnings from check warning tooling --- Tools/build/check_warnings.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 7a0932dc017c53..442ed9b4458a2f 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -93,12 +93,17 @@ def extract_warnings_from_compiler_output_json( def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: """ Returns a dictionary where the key is the file and the data is the warnings - in that file + in that file. Does not include duplicate warnings for a file from list of + provided warnings. """ warnings_by_file = defaultdict(list) + warnings_added = set() for warning in warnings: - warnings_by_file[warning["file"]].append(warning) - + warning_key = f"{warning['file']}-{warning['line']}-{warning['column']}-{warning['option']}" + if warning_key not in warnings_added: + warnings_added.add(warning_key) + warnings_by_file[warning["file"]].append(warning) + return warnings_by_file @@ -113,12 +118,14 @@ def get_unexpected_warnings( """ unexpected_warnings = [] for file in files_with_warnings.keys(): + found_file_in_ignore_list = False for ignore_file in files_with_expected_warnings: if file == ignore_file[0]: if len(files_with_warnings[file]) > ignore_file[1]: unexpected_warnings.extend(files_with_warnings[file]) + found_file_in_ignore_list = True break - if file not in files_with_expected_warnings: + if not found_file_in_ignore_list: unexpected_warnings.extend(files_with_warnings[file]) if unexpected_warnings: From 4164b81406e11618f664f73a4cae4378e245f531 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 5 Aug 2024 19:04:17 -0500 Subject: [PATCH 036/115] Trim trailing whitespace --- Tools/build/check_warnings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 442ed9b4458a2f..b9e80f3bcfea9e 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -103,7 +103,7 @@ def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: if warning_key not in warnings_added: warnings_added.add(warning_key) warnings_by_file[warning["file"]].append(warning) - + return warnings_by_file From 8b07ae8845cf74879eaa3722e9499b461663bb12 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 00:06:24 +0000 Subject: [PATCH 037/115] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by?= =?UTF-8?q?=20blurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Security/2024-08-06-00-06-23.gh-issue-112301.4k4lw6.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Security/2024-08-06-00-06-23.gh-issue-112301.4k4lw6.rst diff --git a/Misc/NEWS.d/next/Security/2024-08-06-00-06-23.gh-issue-112301.4k4lw6.rst b/Misc/NEWS.d/next/Security/2024-08-06-00-06-23.gh-issue-112301.4k4lw6.rst new file mode 100644 index 00000000000000..83fcc91cd09cb0 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-08-06-00-06-23.gh-issue-112301.4k4lw6.rst @@ -0,0 +1 @@ +Add ability to ignore warnings per file with warning count in warning checking tooling. Patch by Nate Ohlson. From 17858b9d61b3613aafa674c6e5cfa76211de0483 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 5 Aug 2024 19:07:38 -0500 Subject: [PATCH 038/115] Reformat news --- .../Security/2024-08-06-00-06-23.gh-issue-112301.4k4lw6.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Security/2024-08-06-00-06-23.gh-issue-112301.4k4lw6.rst b/Misc/NEWS.d/next/Security/2024-08-06-00-06-23.gh-issue-112301.4k4lw6.rst index 83fcc91cd09cb0..0bd2f4d7810a78 100644 --- a/Misc/NEWS.d/next/Security/2024-08-06-00-06-23.gh-issue-112301.4k4lw6.rst +++ b/Misc/NEWS.d/next/Security/2024-08-06-00-06-23.gh-issue-112301.4k4lw6.rst @@ -1 +1,2 @@ -Add ability to ignore warnings per file with warning count in warning checking tooling. Patch by Nate Ohlson. +Add ability to ignore warnings per file with warning count in warning checking tooling. +Patch by Nate Ohlson. From 55e20bdaa8913c4c416f1173587bdd3e60cf04f6 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 6 Aug 2024 14:05:40 -0500 Subject: [PATCH 039/115] Update helpers for function argument types --- Tools/build/check_warnings.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index b9e80f3bcfea9e..8d0dd2218ce475 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -108,7 +108,7 @@ def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: def get_unexpected_warnings( - files_with_expected_warnings: set[str], + files_with_expected_warnings: set[tuple[str, int]], files_with_warnings: dict[str, list[dict]], ) -> int: """ @@ -138,7 +138,7 @@ def get_unexpected_warnings( def get_unexpected_improvements( - files_with_expected_warnings: set[str], + files_with_expected_warnings: set[tuple[str, int]], files_with_warnings: dict[str, list[dict]], ) -> int: """ @@ -230,13 +230,15 @@ def main(argv: list[str] | None = None) -> int: with Path(args.warning_ignore_file_path).open( encoding="UTF-8" ) as clean_files: + # Files with expected warnings are stored as a set of tuples + # where the first element is the file name and the second element + # is the number of warnings expected in that file files_with_expected_warnings = [ (file.strip().split()[0], int(file.strip().split()[1])) for file in clean_files if file.strip() and not file.startswith("#") ] - with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f: compiler_output_file_contents = f.read() From 566e8cd313f1d7589851b9bdca6fd5183d4effa6 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 6 Aug 2024 14:06:37 -0500 Subject: [PATCH 040/115] Format line length --- Tools/build/check_warnings.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 8d0dd2218ce475..b57603efb0a64a 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -49,9 +49,7 @@ def extract_warnings_from_compiler_output_json( """ # Regex to find json arrays at the top level of the file # in the compiler output - json_arrays = re.findall( - r"\[(?:[^[\]]|\[[^\]]*\])*\]", compiler_output - ) + json_arrays = re.findall(r"\[(?:[^[\]]|\[[^\]]*\])*\]", compiler_output) compiler_warnings = [] for array in json_arrays: try: @@ -150,7 +148,7 @@ def get_unexpected_improvements( if file[0] not in files_with_warnings.keys(): unexpected_improvements.append(file) else: - if (len(files_with_warnings[file[0]]) < file[1]): + if len(files_with_warnings[file[0]]) < file[1]: unexpected_improvements.append(file) if unexpected_improvements: From b00cc5121a37584f534e29c52cadd62720026083 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 12 Aug 2024 00:50:58 -0500 Subject: [PATCH 041/115] Add new warnings --- configure | 156 +++++++++++++++++++++++++++++++++++++++++++++++++++ configure.ac | 4 ++ 2 files changed, 160 insertions(+) diff --git a/configure b/configure index a0fbebcb0442b9..e16ba8f152a34a 100755 --- a/configure +++ b/configure @@ -9769,6 +9769,162 @@ then : else $as_nop { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wtrampolines not supported" >&5 printf "%s\n" "$as_me: WARNING: -Wtrampolines not supported" >&2;} +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wimplicit-fallthrough" >&5 +printf %s "checking whether C compiler accepts -Wimplicit-fallthrough... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wimplicit_fallthrough+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wimplicit-fallthrough" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wimplicit_fallthrough=yes +else $as_nop + ax_cv_check_cflags__Werror__Wimplicit_fallthrough=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wimplicit-fallthrough not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wimplicit-fallthrough not supported" >&2;} +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Werror=format-security" >&5 +printf %s "checking whether C compiler accepts -Werror=format-security... " >&6; } +if test ${ax_cv_check_cflags__Werror__Werror_format_security+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Werror=format-security" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Werror_format_security=yes +else $as_nop + ax_cv_check_cflags__Werror__Werror_format_security=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Werror_format_security" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Werror_format_security" >&6; } +if test "x$ax_cv_check_cflags__Werror__Werror_format_security" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Werror=format-security not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Werror=format-security not supported" >&2;} +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wbidi-chars=any" >&5 +printf %s "checking whether C compiler accepts -Wbidi-chars=any... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wbidi_chars_any+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wbidi-chars=any" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wbidi_chars_any=yes +else $as_nop + ax_cv_check_cflags__Werror__Wbidi_chars_any=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wbidi_chars_any" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wbidi_chars_any" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wbidi_chars_any" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wbidi-chars=any not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wbidi-chars=any not supported" >&2;} +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wall" >&5 +printf %s "checking whether C compiler accepts -Wall... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wall+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wall" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wall=yes +else $as_nop + ax_cv_check_cflags__Werror__Wall=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wall" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wall" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wall" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wall" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wall not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wall not supported" >&2;} fi fi diff --git a/configure.ac b/configure.ac index 9a17fc279e5a69..7ce347d7c0b6b5 100644 --- a/configure.ac +++ b/configure.ac @@ -2510,6 +2510,10 @@ if test "$disable_safety" = "no" then AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [CFLAGS_NODIST="$CFLAGS_NODIST -fstack-protector-strong"], [AC_MSG_WARN([-fstack-protector-strong not supported])], [-Werror]) AX_CHECK_COMPILE_FLAG([-Wtrampolines], [CFLAGS_NODIST="$CFLAGS_NODIST -Wtrampolines"], [AC_MSG_WARN([-Wtrampolines not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough"], [AC_MSG_WARN([-Wimplicit-fallthrough not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Werror=format-security], [CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security"], [AC_MSG_WARN([-Werror=format-security not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wbidi-chars=any], [CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any"], [AC_MSG_WARN([-Wbidi-chars=any not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wall], [CFLAGS_NODIST="$CFLAGS_NODIST -Wall"], [AC_MSG_WARN([-Wall not supported])], [-Werror]) fi AC_MSG_CHECKING([for --enable-slower-safety]) From f8ae75ea892eaa5314e0c645bad13cfc0cb2bc51 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 12 Aug 2024 17:00:15 -0500 Subject: [PATCH 042/115] Use named tuple for warning filename and count --- Tools/build/check_warnings.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index b57603efb0a64a..72fb8a6178d294 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -9,6 +9,11 @@ import re import sys from pathlib import Path +from typing import NamedTuple + +class FileWarnings(NamedTuple): + name: str + count: int def extract_warnings_from_compiler_output_clang( @@ -49,7 +54,7 @@ def extract_warnings_from_compiler_output_json( """ # Regex to find json arrays at the top level of the file # in the compiler output - json_arrays = re.findall(r"\[(?:[^[\]]|\[[^\]]*\])*\]", compiler_output) + json_arrays = re.findall(r"\[(?:[^[\]]|\[[^]]*])*]", compiler_output) compiler_warnings = [] for array in json_arrays: try: @@ -97,7 +102,10 @@ def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: warnings_by_file = defaultdict(list) warnings_added = set() for warning in warnings: - warning_key = f"{warning['file']}-{warning['line']}-{warning['column']}-{warning['option']}" + warning_key = ( + f"{warning['file']}-{warning['line']}-" + f"{warning['column']}-{warning['option']}" + ) if warning_key not in warnings_added: warnings_added.add(warning_key) warnings_by_file[warning["file"]].append(warning) @@ -118,8 +126,8 @@ def get_unexpected_warnings( for file in files_with_warnings.keys(): found_file_in_ignore_list = False for ignore_file in files_with_expected_warnings: - if file == ignore_file[0]: - if len(files_with_warnings[file]) > ignore_file[1]: + if file == ignore_file.name: + if len(files_with_warnings[file]) > ignore_file.count: unexpected_warnings.extend(files_with_warnings[file]) found_file_in_ignore_list = True break @@ -145,10 +153,10 @@ def get_unexpected_improvements( """ unexpected_improvements = [] for file in files_with_expected_warnings: - if file[0] not in files_with_warnings.keys(): + if file.name not in files_with_warnings.keys(): unexpected_improvements.append(file) else: - if len(files_with_warnings[file[0]]) < file[1]: + if len(files_with_warnings[file.name]) < file.count: unexpected_improvements.append(file) if unexpected_improvements: @@ -232,7 +240,7 @@ def main(argv: list[str] | None = None) -> int: # where the first element is the file name and the second element # is the number of warnings expected in that file files_with_expected_warnings = [ - (file.strip().split()[0], int(file.strip().split()[1])) + FileWarning(file.strip().split()[0], int(file.strip().split()[1])) for file in clean_files if file.strip() and not file.startswith("#") ] From f68eb713bb11f6a9852d8cfd31ebc8ac27a4bd3f Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 12 Aug 2024 17:22:47 -0500 Subject: [PATCH 043/115] Add more warnings --- configure | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++ configure.ac | 2 ++ 2 files changed, 80 insertions(+) diff --git a/configure b/configure index e16ba8f152a34a..eb9edcec7fbae8 100755 --- a/configure +++ b/configure @@ -9769,6 +9769,45 @@ then : else $as_nop { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wtrampolines not supported" >&5 printf "%s\n" "$as_me: WARNING: -Wtrampolines not supported" >&2;} +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wconversion" >&5 +printf %s "checking whether C compiler accepts -Wconversion... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wconversion+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wconversion" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wconversion=yes +else $as_nop + ax_cv_check_cflags__Werror__Wconversion=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wconversion" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wconversion" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wconversion" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wconversion" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wconversion not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wconversion not supported" >&2;} fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wimplicit-fallthrough" >&5 @@ -9847,6 +9886,45 @@ then : else $as_nop { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Werror=format-security not supported" >&5 printf "%s\n" "$as_me: WARNING: -Werror=format-security not supported" >&2;} +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wformat=2" >&5 +printf %s "checking whether C compiler accepts -Wformat=2... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wformat_2+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wformat=2" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wformat_2=yes +else $as_nop + ax_cv_check_cflags__Werror__Wformat_2=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wformat_2" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wformat_2" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wformat_2" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wformat=2" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wformat=2 not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wformat=2 not supported" >&2;} fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wbidi-chars=any" >&5 diff --git a/configure.ac b/configure.ac index 7ce347d7c0b6b5..63fe67c96e8d81 100644 --- a/configure.ac +++ b/configure.ac @@ -2510,8 +2510,10 @@ if test "$disable_safety" = "no" then AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [CFLAGS_NODIST="$CFLAGS_NODIST -fstack-protector-strong"], [AC_MSG_WARN([-fstack-protector-strong not supported])], [-Werror]) AX_CHECK_COMPILE_FLAG([-Wtrampolines], [CFLAGS_NODIST="$CFLAGS_NODIST -Wtrampolines"], [AC_MSG_WARN([-Wtrampolines not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wconversion], [CFLAGS_NODIST="$CFLAGS_NODIST -Wconversion"], [AC_MSG_WARN([-Wconversion not supported])], [-Werror]) AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough"], [AC_MSG_WARN([-Wimplicit-fallthrough not supported])], [-Werror]) AX_CHECK_COMPILE_FLAG([-Werror=format-security], [CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security"], [AC_MSG_WARN([-Werror=format-security not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wformat=2], [CFLAGS_NODIST="$CFLAGS_NODIST -Wformat=2"], [AC_MSG_WARN([-Wformat=2 not supported])], [-Werror]) AX_CHECK_COMPILE_FLAG([-Wbidi-chars=any], [CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any"], [AC_MSG_WARN([-Wbidi-chars=any not supported])], [-Werror]) AX_CHECK_COMPILE_FLAG([-Wall], [CFLAGS_NODIST="$CFLAGS_NODIST -Wall"], [AC_MSG_WARN([-Wall not supported])], [-Werror]) fi From cbf069b461c5b32996aff6d6946a21fdbb85b4d1 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Mon, 12 Aug 2024 18:10:30 -0500 Subject: [PATCH 044/115] Use proper type --- Tools/build/check_warnings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 72fb8a6178d294..2052b42d529672 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -162,7 +162,7 @@ def get_unexpected_improvements( if unexpected_improvements: print("Unexpected improvements:") for file in unexpected_improvements: - print(file) + print(file.name) return 1 return 0 @@ -240,7 +240,7 @@ def main(argv: list[str] | None = None) -> int: # where the first element is the file name and the second element # is the number of warnings expected in that file files_with_expected_warnings = [ - FileWarning(file.strip().split()[0], int(file.strip().split()[1])) + FileWarnings(file.strip().split()[0], int(file.strip().split()[1])) for file in clean_files if file.strip() and not file.startswith("#") ] From e8ef3e359b6197a2de5cdf4d085a651205e93e5f Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 00:17:30 -0500 Subject: [PATCH 045/115] Add path prefix argument to check warnings script --- Tools/build/.warningignore_ubuntu | 234 ++++++++++++++++++++++++++++++ Tools/build/check_warnings.py | 15 +- 2 files changed, 246 insertions(+), 3 deletions(-) diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 469c727abfb11c..742858dbb10975 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -3,3 +3,237 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) +Include/internal/pycore_gc.h 1 +Include/internal/pycore_backoff.h 3 +Include/internal/pycore_list.h 1 +Include/internal/mimalloc/mimalloc/internal.h 4 +Parser/pegen.c 8 +Parser/string_parser.c 7 +Parser/parser.c 116 +Parser/action_helpers.c 3 +Parser/lexer/buffer.c 1 +Parser/lexer/lexer.c 14 +Parser/tokenizer/readline_tokenizer.c 4 +Parser/tokenizer/file_tokenizer.c 9 +Parser/tokenizer/helpers.c 8 +Include/internal/pycore_object.h 3 +Include/internal/pycore_long.h 3 +Objects/abstract.c 6 +Objects/bytes_methods.c 9 +Objects/stringlib/fastsearch.h 8 +Objects/bytearrayobject.c 46 +Objects/stringlib/join.h 4 +Objects/stringlib/transmogrify.h 27 +Objects/bytesobject.c 50 +Include/internal/pycore_dict.h 2 +Objects/call.c 13 +Objects/classobject.c 4 +Objects/codeobject.c 25 +Objects/descrobject.c 2 +Objects/genobject.c 3 +Objects/fileobject.c 3 +Objects/floatobject.c 18 +Objects/frameobject.c 18 +Objects/funcobject.c 2 +Objects/listobject.c 42 +Objects/longobject.c 58 +Objects/stringlib/eq.h 1 +Objects/dictobject.c 35 +Objects/odictobject.c 7 +Objects/memoryobject.c 12 +Objects/methodobject.c 1 +Objects/moduleobject.c 4 +Objects/object.c 1 +Objects/mimalloc/alloc.c 6 +Objects/mimalloc/arena.c 6 +Objects/mimalloc/heap.c 2 +Objects/mimalloc/init.c 2 +Objects/mimalloc/options.c 6 +Objects/mimalloc/os.c 4 +Objects/mimalloc/page-queue.c 2 +Objects/mimalloc/page.c 2 +Objects/mimalloc/random.c 1 +Objects/mimalloc/segment.c 11 +Objects/mimalloc/stats.c 5 +Objects/mimalloc/prim/unix/prim.c 6 +Objects/obmalloc.c 7 +Objects/rangeobject.c 10 +Objects/setobject.c 13 +Objects/sliceobject.c 4 +Objects/structseq.c 13 +Objects/tupleobject.c 8 +Objects/typeobject.c 42 +Objects/unicodeobject.c 155 +Objects/stringlib/replace.h 5 +Objects/stringlib/repr.h 21 +Objects/stringlib/codecs.h 12 +Objects/unicodectype.c 7 +Python/Python-ast.c 14 +Python/asdl.c 3 +Python/assemble.c 11 +Python/ast_opt.c 5 +Python/bltinmodule.c 8 +Python/ceval.c 8 +Python/generated_cases.c.h 23 +Python/codecs.c 28 +Python/compile.c 8 +Python/context.c 1 +Python/crossinterp_data_lookup.h 1 +Python/crossinterp.c 2 +Python/errors.c 1 +Python/flowgraph.c 8 +Python/frame.c 3 +Python/gc.c 8 +Python/getargs.c 13 +Python/getversion.c 1 +Python/ceval_gil.c 2 +Python/hashtable.c 1 +Python/import.c 6 +Python/initconfig.c 3 +Python/instrumentation.c 42 +Python/intrinsics.c 1 +Python/lock.c 4 +Python/legacy_tracing.c 3 +Python/marshal.c 17 +Python/modsupport.c 3 +Python/mystrtoul.c 4 +Python/pathconfig.c 1 +Python/pyarena.c 1 +Python/preconfig.c 2 +Python/pyhash.c 4 +Python/pylifecycle.c 7 +Python/pystate.c 6 +Python/pytime.c 2 +Python/qsbr.c 2 +Python/bootstrap_hash.c 7 +Python/specialize.c 7 +Python/symtable.c 18 +Python/sysmodule.c 3 +Python/thread_pthread.h 6 +Python/thread.c 1 +Python/traceback.c 6 +Python/tracemalloc.c 6 +Python/pystrtod.c 14 +Python/pystrhex.c 18 +Python/dtoa.c 32 +Python/formatter_unicode.c 6 +Python/fileutils.c 11 +Python/suggestions.c 12 +Python/perf_trampoline.c 12 +Python/perf_jit_trampoline.c 32 +Modules/main.c 2 +Modules/atexitmodule.c 1 +Modules/faulthandler.c 5 +Modules/signalmodule.c 3 +Modules/posixmodule.c 89 +Modules/_collectionsmodule.c 2 +Modules/_io/_iomodule.c 1 +Modules/_io/iobase.c 1 +Modules/_io/fileio.c 9 +Modules/_io/bytesio.c 15 +Modules/_io/bufferedio.c 16 +Modules/_io/textio.c 17 +Modules/_io/stringio.c 8 +Modules/itertoolsmodule.c 9 +Modules/timemodule.c 10 +Modules/_functoolsmodule.c 6 +Modules/_localemodule.c 3 +Modules/_operator.c 5 +Modules/pwdmodule.c 4 +Programs/_freeze_module.c 1 +Modules/clinic/arraymodule.c.h 1 +Modules/arraymodule.c 50 +Modules/_asynciomodule.c 3 +Modules/_bisectmodule.c 4 +Modules/_csv.c 3 +Modules/_json.c 19 +Modules/_lsprof.c 5 +Modules/rotatingtree.c 2 +Modules/_pickle.c 75 +Modules/_queuemodule.c 4 +Modules/_randommodule.c 3 +Modules/_struct.c 4 +Modules/_interpqueuesmodule.c 1 +Modules/_interpchannelsmodule.c 1 +Modules/_testexternalinspection.c 6 +Modules/_zoneinfo.c 17 +Modules/cmathmodule.c 1 +Modules/mathmodule.c 15 +Modules/_datetimemodule.c 34 +Include/internal/pycore_blocks_output_buffer.h 2 +Modules/_bz2module.c 5 +Modules/_decimal/_decimal.c 19 +Modules/binascii.c 208 +Modules/zlibmodule.c 26 +Modules/_lzmamodule.c 6 +Modules/_dbmmodule.c 8 +Modules/_gdbmmodule.c 5 +Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4 +Modules/readline.c 1 +Modules/sha3module.c 4 +Modules/_blake2/blake2b_impl.c 12 +Modules/_blake2/blake2s_impl.c 12 +Modules/expat/siphash.h 1 +Modules/expat/xmlparse.c 46 +Modules/expat/xmltok.c 13 +Modules/expat/xmltok_impl.c 8 +Modules/cjkcodecs/_codecs_cn.c 11 +Modules/cjkcodecs/_codecs_hk.c 2 +Modules/cjkcodecs/_codecs_iso2022.c 10 +Modules/cjkcodecs/_codecs_jp.c 32 +Modules/cjkcodecs/_codecs_kr.c 15 +Modules/cjkcodecs/_codecs_tw.c 4 +Modules/cjkcodecs/multibytecodec.c 13 +Modules/fcntlmodule.c 6 +Modules/unicodedata_db.h 1 +Modules/unicodedata.c 28 +Modules/clinic/unicodedata.c.h 10 +Modules/grpmodule.c 4 +Modules/mmapmodule.c 20 +Modules/resource.c 5 +Modules/_posixsubprocess.c 12 +Modules/selectmodule.c 4 +Modules/socketmodule.c 81 +Modules/termios.c 1 +Modules/syslogmodule.c 3 +Modules/_multiprocessing/posixshmem.c 1 +Modules/_multiprocessing/semaphore.c 1 +Modules/_ctypes/callbacks.c 4 +Modules/_ctypes/_ctypes.c 55 +Modules/_ctypes/callproc.c 13 +Modules/_ctypes/stgdict.c 19 +Modules/_ctypes/cfield.c 61 +Modules/_cursesmodule.c 28 +Modules/_sqlite/connection.c 5 +Modules/_sqlite/cursor.c 3 +Modules/_sqlite/module.c 2 +Modules/_ssl.c 31 +Modules/_hashopenssl.c 16 +Modules/_tkinter.c 10 +Modules/_xxtestfuzz/_xxtestfuzz.c 1 +Modules/_xxtestfuzz/fuzzer.c 13 +Modules/_testbuffer.c 28 +Modules/_testinternalcapi/test_lock.c 4 +Modules/_testinternalcapi.c 10 +Modules/_testinternalcapi/test_critical_sections.c 1 +Modules/_testcapimodule.c 3 +Modules/_testcapi/vectorcall.c 3 +Modules/_testcapi/heaptype.c 1 +Modules/_testcapi/unicode.c 2 +Modules/_testcapi/mem.c 2 +Modules/_testcapi/watchers.c 3 +Modules/_testcapi/long.c 2 +Modules/_testcapi/pyatomic.c 1 +Modules/_testcapi/bytes.c 1 +Modules/_testcapi/monitoring.c 3 +Modules/_testlimitedcapi/heaptype_relative.c 3 +Modules/_testlimitedcapi/object.c 2 +Modules/_testlimitedcapi/unicode.c 2 +Modules/clinic/_testclinic.c.h 1 +Modules/_testclinic.c 1 +Modules/_testmultiphase.c 1 +Modules/pyexpat.c 14 +Modules/_elementtree.c 43 +Modules/getpath.c 7 +Modules/_sre/sre.c 16 +Modules/_sre/sre_lib.h 62 diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 2052b42d529672..a48e49ebbab641 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -43,6 +43,7 @@ def extract_warnings_from_compiler_output_clang( def extract_warnings_from_compiler_output_json( compiler_output: str, + path_prefix: str | None = None, ) -> list[dict]: """ Extracts warnings from the compiler output when using @@ -73,7 +74,7 @@ def extract_warnings_from_compiler_output_json( compiler_warnings.append( { # Remove leading current directory if present - "file": location[key]["file"].lstrip("./"), + "file": location[key]["file"].lstrip(path_prefix), "line": location[key]["line"], "column": location[key]["column"], "message": warning["message"], @@ -206,6 +207,12 @@ def main(argv: list[str] | None = None) -> int: choices=["json", "clang"], help="Type of compiler output file (json or clang)", ) + parser.add_argument( + "-p", + "--path-prefix", + type=str, + help="Path prefix to remove from the start of file paths", + ) args = parser.parse_args(argv) @@ -250,11 +257,13 @@ def main(argv: list[str] | None = None) -> int: if args.compiler_output_type == "json": warnings = extract_warnings_from_compiler_output_json( - compiler_output_file_contents + compiler_output_file_contents, + args.path_prefix ) elif args.compiler_output_type == "clang": warnings = extract_warnings_from_compiler_output_clang( - compiler_output_file_contents + compiler_output_file_contents, + args.path_prefix ) files_with_warnings = get_warnings_by_file(warnings) From ba6486f440aa11eeed8b5e6766db21df13e0520e Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 01:09:32 -0500 Subject: [PATCH 046/115] Add path prefix to check warning script call --- .github/workflows/reusable-change-detection.yml | 1 + .github/workflows/reusable-ubuntu.yml | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index 25c789d335efc8..979a958d0d827e 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -140,6 +140,7 @@ jobs: Tools/msi/** .github/workflows/reusable-windows-msi.yml format: csv # works for paths with spaces + contineu-on-error: true - name: Check for changes in MSI installer-related files if: >- steps.changed-win-msi-files.outputs.added_modified_renamed != '' diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 92069fddc31217..766b15d03c49df 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -80,7 +80,14 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo - name: Check compiler warnings - run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json + run: >- + python Tools/build/check_warnings.py + --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt + --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu + --compiler-output-type=json + --fail-on-regression + --fail-on-improvement + --path-prefix="../cpython-ro-srcdir/" - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw From 034552f2ab046c9f07bb93aaeee979822b0cfdaf Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 01:10:32 -0500 Subject: [PATCH 047/115] Fix spelling --- .github/workflows/reusable-change-detection.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index 979a958d0d827e..b1733aae411e1e 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -140,7 +140,7 @@ jobs: Tools/msi/** .github/workflows/reusable-windows-msi.yml format: csv # works for paths with spaces - contineu-on-error: true + continue-on-error: true - name: Check for changes in MSI installer-related files if: >- steps.changed-win-msi-files.outputs.added_modified_renamed != '' From 9b724b3c8fd9f141d87cfea1103f7d28282bd1d2 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 01:24:33 -0500 Subject: [PATCH 048/115] Add condition to check MSI files only on PR --- .github/workflows/reusable-change-detection.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index b1733aae411e1e..7880165d6dfc60 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -134,13 +134,13 @@ jobs: echo "run-docs=true" >> "${GITHUB_OUTPUT}" - name: Get a list of the MSI installer-related files id: changed-win-msi-files + if: github.event_name == 'pull_request' uses: Ana06/get-changed-files@v2.3.0 with: filter: | Tools/msi/** .github/workflows/reusable-windows-msi.yml format: csv # works for paths with spaces - continue-on-error: true - name: Check for changes in MSI installer-related files if: >- steps.changed-win-msi-files.outputs.added_modified_renamed != '' From ab8f099c3512e4a8137e521ea93423c7165e9b9b Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 01:31:29 -0500 Subject: [PATCH 049/115] Add warnings to ubuntu ignore list --- Tools/build/.warningignore_ubuntu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 742858dbb10975..5d54c64c06d555 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -202,6 +202,8 @@ Modules/_ctypes/callbacks.c 4 Modules/_ctypes/_ctypes.c 55 Modules/_ctypes/callproc.c 13 Modules/_ctypes/stgdict.c 19 +Modules/_ctypes/_ctypes_test.c 7 +Modules/_ctypes/_ctypes_test_generated.c.h 363 Modules/_ctypes/cfield.c 61 Modules/_cursesmodule.c 28 Modules/_sqlite/connection.c 5 From 6296a46dfb965742c1c092ac63c529ea6bf5bdb4 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 01:48:27 -0500 Subject: [PATCH 050/115] Remove warning ignore files --- Tools/build/.warningignore_ubuntu | 236 ------------------------------ 1 file changed, 236 deletions(-) diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 5d54c64c06d555..469c727abfb11c 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -3,239 +3,3 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) -Include/internal/pycore_gc.h 1 -Include/internal/pycore_backoff.h 3 -Include/internal/pycore_list.h 1 -Include/internal/mimalloc/mimalloc/internal.h 4 -Parser/pegen.c 8 -Parser/string_parser.c 7 -Parser/parser.c 116 -Parser/action_helpers.c 3 -Parser/lexer/buffer.c 1 -Parser/lexer/lexer.c 14 -Parser/tokenizer/readline_tokenizer.c 4 -Parser/tokenizer/file_tokenizer.c 9 -Parser/tokenizer/helpers.c 8 -Include/internal/pycore_object.h 3 -Include/internal/pycore_long.h 3 -Objects/abstract.c 6 -Objects/bytes_methods.c 9 -Objects/stringlib/fastsearch.h 8 -Objects/bytearrayobject.c 46 -Objects/stringlib/join.h 4 -Objects/stringlib/transmogrify.h 27 -Objects/bytesobject.c 50 -Include/internal/pycore_dict.h 2 -Objects/call.c 13 -Objects/classobject.c 4 -Objects/codeobject.c 25 -Objects/descrobject.c 2 -Objects/genobject.c 3 -Objects/fileobject.c 3 -Objects/floatobject.c 18 -Objects/frameobject.c 18 -Objects/funcobject.c 2 -Objects/listobject.c 42 -Objects/longobject.c 58 -Objects/stringlib/eq.h 1 -Objects/dictobject.c 35 -Objects/odictobject.c 7 -Objects/memoryobject.c 12 -Objects/methodobject.c 1 -Objects/moduleobject.c 4 -Objects/object.c 1 -Objects/mimalloc/alloc.c 6 -Objects/mimalloc/arena.c 6 -Objects/mimalloc/heap.c 2 -Objects/mimalloc/init.c 2 -Objects/mimalloc/options.c 6 -Objects/mimalloc/os.c 4 -Objects/mimalloc/page-queue.c 2 -Objects/mimalloc/page.c 2 -Objects/mimalloc/random.c 1 -Objects/mimalloc/segment.c 11 -Objects/mimalloc/stats.c 5 -Objects/mimalloc/prim/unix/prim.c 6 -Objects/obmalloc.c 7 -Objects/rangeobject.c 10 -Objects/setobject.c 13 -Objects/sliceobject.c 4 -Objects/structseq.c 13 -Objects/tupleobject.c 8 -Objects/typeobject.c 42 -Objects/unicodeobject.c 155 -Objects/stringlib/replace.h 5 -Objects/stringlib/repr.h 21 -Objects/stringlib/codecs.h 12 -Objects/unicodectype.c 7 -Python/Python-ast.c 14 -Python/asdl.c 3 -Python/assemble.c 11 -Python/ast_opt.c 5 -Python/bltinmodule.c 8 -Python/ceval.c 8 -Python/generated_cases.c.h 23 -Python/codecs.c 28 -Python/compile.c 8 -Python/context.c 1 -Python/crossinterp_data_lookup.h 1 -Python/crossinterp.c 2 -Python/errors.c 1 -Python/flowgraph.c 8 -Python/frame.c 3 -Python/gc.c 8 -Python/getargs.c 13 -Python/getversion.c 1 -Python/ceval_gil.c 2 -Python/hashtable.c 1 -Python/import.c 6 -Python/initconfig.c 3 -Python/instrumentation.c 42 -Python/intrinsics.c 1 -Python/lock.c 4 -Python/legacy_tracing.c 3 -Python/marshal.c 17 -Python/modsupport.c 3 -Python/mystrtoul.c 4 -Python/pathconfig.c 1 -Python/pyarena.c 1 -Python/preconfig.c 2 -Python/pyhash.c 4 -Python/pylifecycle.c 7 -Python/pystate.c 6 -Python/pytime.c 2 -Python/qsbr.c 2 -Python/bootstrap_hash.c 7 -Python/specialize.c 7 -Python/symtable.c 18 -Python/sysmodule.c 3 -Python/thread_pthread.h 6 -Python/thread.c 1 -Python/traceback.c 6 -Python/tracemalloc.c 6 -Python/pystrtod.c 14 -Python/pystrhex.c 18 -Python/dtoa.c 32 -Python/formatter_unicode.c 6 -Python/fileutils.c 11 -Python/suggestions.c 12 -Python/perf_trampoline.c 12 -Python/perf_jit_trampoline.c 32 -Modules/main.c 2 -Modules/atexitmodule.c 1 -Modules/faulthandler.c 5 -Modules/signalmodule.c 3 -Modules/posixmodule.c 89 -Modules/_collectionsmodule.c 2 -Modules/_io/_iomodule.c 1 -Modules/_io/iobase.c 1 -Modules/_io/fileio.c 9 -Modules/_io/bytesio.c 15 -Modules/_io/bufferedio.c 16 -Modules/_io/textio.c 17 -Modules/_io/stringio.c 8 -Modules/itertoolsmodule.c 9 -Modules/timemodule.c 10 -Modules/_functoolsmodule.c 6 -Modules/_localemodule.c 3 -Modules/_operator.c 5 -Modules/pwdmodule.c 4 -Programs/_freeze_module.c 1 -Modules/clinic/arraymodule.c.h 1 -Modules/arraymodule.c 50 -Modules/_asynciomodule.c 3 -Modules/_bisectmodule.c 4 -Modules/_csv.c 3 -Modules/_json.c 19 -Modules/_lsprof.c 5 -Modules/rotatingtree.c 2 -Modules/_pickle.c 75 -Modules/_queuemodule.c 4 -Modules/_randommodule.c 3 -Modules/_struct.c 4 -Modules/_interpqueuesmodule.c 1 -Modules/_interpchannelsmodule.c 1 -Modules/_testexternalinspection.c 6 -Modules/_zoneinfo.c 17 -Modules/cmathmodule.c 1 -Modules/mathmodule.c 15 -Modules/_datetimemodule.c 34 -Include/internal/pycore_blocks_output_buffer.h 2 -Modules/_bz2module.c 5 -Modules/_decimal/_decimal.c 19 -Modules/binascii.c 208 -Modules/zlibmodule.c 26 -Modules/_lzmamodule.c 6 -Modules/_dbmmodule.c 8 -Modules/_gdbmmodule.c 5 -Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4 -Modules/readline.c 1 -Modules/sha3module.c 4 -Modules/_blake2/blake2b_impl.c 12 -Modules/_blake2/blake2s_impl.c 12 -Modules/expat/siphash.h 1 -Modules/expat/xmlparse.c 46 -Modules/expat/xmltok.c 13 -Modules/expat/xmltok_impl.c 8 -Modules/cjkcodecs/_codecs_cn.c 11 -Modules/cjkcodecs/_codecs_hk.c 2 -Modules/cjkcodecs/_codecs_iso2022.c 10 -Modules/cjkcodecs/_codecs_jp.c 32 -Modules/cjkcodecs/_codecs_kr.c 15 -Modules/cjkcodecs/_codecs_tw.c 4 -Modules/cjkcodecs/multibytecodec.c 13 -Modules/fcntlmodule.c 6 -Modules/unicodedata_db.h 1 -Modules/unicodedata.c 28 -Modules/clinic/unicodedata.c.h 10 -Modules/grpmodule.c 4 -Modules/mmapmodule.c 20 -Modules/resource.c 5 -Modules/_posixsubprocess.c 12 -Modules/selectmodule.c 4 -Modules/socketmodule.c 81 -Modules/termios.c 1 -Modules/syslogmodule.c 3 -Modules/_multiprocessing/posixshmem.c 1 -Modules/_multiprocessing/semaphore.c 1 -Modules/_ctypes/callbacks.c 4 -Modules/_ctypes/_ctypes.c 55 -Modules/_ctypes/callproc.c 13 -Modules/_ctypes/stgdict.c 19 -Modules/_ctypes/_ctypes_test.c 7 -Modules/_ctypes/_ctypes_test_generated.c.h 363 -Modules/_ctypes/cfield.c 61 -Modules/_cursesmodule.c 28 -Modules/_sqlite/connection.c 5 -Modules/_sqlite/cursor.c 3 -Modules/_sqlite/module.c 2 -Modules/_ssl.c 31 -Modules/_hashopenssl.c 16 -Modules/_tkinter.c 10 -Modules/_xxtestfuzz/_xxtestfuzz.c 1 -Modules/_xxtestfuzz/fuzzer.c 13 -Modules/_testbuffer.c 28 -Modules/_testinternalcapi/test_lock.c 4 -Modules/_testinternalcapi.c 10 -Modules/_testinternalcapi/test_critical_sections.c 1 -Modules/_testcapimodule.c 3 -Modules/_testcapi/vectorcall.c 3 -Modules/_testcapi/heaptype.c 1 -Modules/_testcapi/unicode.c 2 -Modules/_testcapi/mem.c 2 -Modules/_testcapi/watchers.c 3 -Modules/_testcapi/long.c 2 -Modules/_testcapi/pyatomic.c 1 -Modules/_testcapi/bytes.c 1 -Modules/_testcapi/monitoring.c 3 -Modules/_testlimitedcapi/heaptype_relative.c 3 -Modules/_testlimitedcapi/object.c 2 -Modules/_testlimitedcapi/unicode.c 2 -Modules/clinic/_testclinic.c.h 1 -Modules/_testclinic.c 1 -Modules/_testmultiphase.c 1 -Modules/pyexpat.c 14 -Modules/_elementtree.c 43 -Modules/getpath.c 7 -Modules/_sre/sre.c 16 -Modules/_sre/sre_lib.h 62 From 19df8034a023711a8ef551ef469e69ebc64a20d1 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 01:56:00 -0500 Subject: [PATCH 051/115] Add files to ignore warnings --- Tools/build/.warningignore_ubuntu | 233 ++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 469c727abfb11c..04e0893b069d5b 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -3,3 +3,236 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) +Include/internal/pycore_gc.h 1 +Include/internal/pycore_backoff.h 3 +Include/internal/pycore_list.h 1 +Include/internal/mimalloc/mimalloc/internal.h 4 +Parser/pegen.c 8 +Parser/string_parser.c 7 +Parser/parser.c 116 +Parser/action_helpers.c 3 +Parser/lexer/buffer.c 1 +Parser/lexer/lexer.c 14 +Parser/tokenizer/readline_tokenizer.c 4 +Parser/tokenizer/file_tokenizer.c 9 +Parser/tokenizer/helpers.c 8 +Include/internal/pycore_object.h 3 +Include/internal/pycore_long.h 3 +Objects/abstract.c 6 +Objects/bytes_methods.c 9 +Objects/stringlib/fastsearch.h 8 +Objects/bytearrayobject.c 46 +Objects/stringlib/join.h 4 +Objects/stringlib/transmogrify.h 27 +Objects/bytesobject.c 50 +Include/internal/pycore_dict.h 2 +Objects/call.c 13 +Objects/classobject.c 4 +Objects/codeobject.c 25 +Objects/descrobject.c 2 +Objects/genobject.c 3 +Objects/fileobject.c 3 +Objects/floatobject.c 18 +Objects/frameobject.c 18 +Objects/funcobject.c 2 +Objects/listobject.c 42 +Objects/longobject.c 58 +Objects/stringlib/eq.h 1 +Objects/dictobject.c 35 +Objects/odictobject.c 7 +Objects/memoryobject.c 12 +Objects/methodobject.c 1 +Objects/moduleobject.c 4 +Objects/object.c 1 +Objects/mimalloc/alloc.c 6 +Objects/mimalloc/arena.c 6 +Objects/mimalloc/heap.c 2 +Objects/mimalloc/init.c 2 +Objects/mimalloc/options.c 6 +Objects/mimalloc/os.c 4 +Objects/mimalloc/page-queue.c 2 +Objects/mimalloc/page.c 2 +Objects/mimalloc/random.c 1 +Objects/mimalloc/segment.c 11 +Objects/mimalloc/stats.c 5 +Objects/mimalloc/prim/unix/prim.c 6 +Objects/obmalloc.c 7 +Objects/rangeobject.c 10 +Objects/setobject.c 13 +Objects/sliceobject.c 4 +Objects/structseq.c 13 +Objects/tupleobject.c 8 +Objects/typeobject.c 42 +Objects/unicodeobject.c 155 +Objects/stringlib/replace.h 5 +Objects/stringlib/repr.h 21 +Objects/stringlib/codecs.h 12 +Objects/unicodectype.c 7 +Python/Python-ast.c 14 +Python/asdl.c 3 +Python/assemble.c 11 +Python/ast_opt.c 5 +Python/bltinmodule.c 8 +Python/ceval.c 8 +Python/generated_cases.c.h 23 +Python/codecs.c 28 +Python/compile.c 8 +Python/context.c 1 +Python/crossinterp_data_lookup.h 1 +Python/crossinterp.c 2 +Python/errors.c 1 +Python/flowgraph.c 8 +Python/frame.c 3 +Python/gc.c 8 +Python/getargs.c 13 +Python/getversion.c 1 +Python/ceval_gil.c 2 +Python/hashtable.c 1 +Python/import.c 6 +Python/initconfig.c 3 +Python/instrumentation.c 42 +Python/intrinsics.c 1 +Python/lock.c 4 +Python/legacy_tracing.c 3 +Python/marshal.c 17 +Python/modsupport.c 3 +Python/mystrtoul.c 4 +Python/pathconfig.c 1 +Python/pyarena.c 1 +Python/preconfig.c 2 +Python/pyhash.c 4 +Python/pylifecycle.c 7 +Python/pystate.c 6 +Python/pytime.c 2 +Python/qsbr.c 2 +Python/bootstrap_hash.c 7 +Python/specialize.c 7 +Python/symtable.c 18 +Python/sysmodule.c 3 +Python/thread_pthread.h 6 +Python/thread.c 1 +Python/traceback.c 6 +Python/tracemalloc.c 6 +Python/pystrtod.c 14 +Python/pystrhex.c 18 +Python/dtoa.c 32 +Python/formatter_unicode.c 6 +Python/fileutils.c 11 +Python/suggestions.c 12 +Python/perf_trampoline.c 12 +Python/perf_jit_trampoline.c 32 +Modules/main.c 2 +Modules/atexitmodule.c 1 +Modules/faulthandler.c 5 +Modules/signalmodule.c 3 +Modules/posixmodule.c 89 +Modules/_collectionsmodule.c 2 +Modules/_io/_iomodule.c 1 +Modules/_io/iobase.c 1 +Modules/_io/fileio.c 9 +Modules/_io/bytesio.c 15 +Modules/_io/bufferedio.c 16 +Modules/_io/textio.c 17 +Modules/_io/stringio.c 8 +Modules/itertoolsmodule.c 9 +Modules/timemodule.c 10 +Modules/_functoolsmodule.c 6 +Modules/_localemodule.c 3 +Modules/_operator.c 5 +Modules/pwdmodule.c 4 +Programs/_freeze_module.c 1 +Modules/clinic/arraymodule.c.h 1 +Modules/arraymodule.c 50 +Modules/_asynciomodule.c 3 +Modules/_bisectmodule.c 4 +Modules/_csv.c 3 +Modules/_json.c 19 +Modules/_lsprof.c 5 +Modules/rotatingtree.c 2 +Modules/_pickle.c 75 +Modules/_queuemodule.c 4 +Modules/_randommodule.c 3 +Modules/_struct.c 4 +Modules/_interpqueuesmodule.c 1 +Modules/_interpchannelsmodule.c 1 +Modules/_zoneinfo.c 17 +Modules/cmathmodule.c 1 +Modules/mathmodule.c 15 +Modules/_datetimemodule.c 34 +Include/internal/pycore_blocks_output_buffer.h 2 +Modules/_bz2module.c 5 +Modules/_decimal/_decimal.c 19 +Modules/binascii.c 208 +Modules/zlibmodule.c 26 +Modules/_lzmamodule.c 6 +Modules/_dbmmodule.c 8 +Modules/_gdbmmodule.c 5 +Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4 +Modules/readline.c 1 +Modules/sha3module.c 4 +Modules/_blake2/blake2b_impl.c 12 +Modules/_blake2/blake2s_impl.c 12 +Modules/expat/siphash.h 1 +Modules/expat/xmlparse.c 46 +Modules/expat/xmltok.c 13 +Modules/expat/xmltok_impl.c 8 +Modules/cjkcodecs/_codecs_cn.c 11 +Modules/cjkcodecs/_codecs_hk.c 2 +Modules/cjkcodecs/_codecs_iso2022.c 10 +Modules/cjkcodecs/_codecs_jp.c 32 +Modules/cjkcodecs/_codecs_kr.c 15 +Modules/cjkcodecs/_codecs_tw.c 4 +Modules/cjkcodecs/multibytecodec.c 13 +Modules/fcntlmodule.c 6 +Modules/unicodedata_db.h 1 +Modules/unicodedata.c 28 +Modules/clinic/unicodedata.c.h 10 +Modules/grpmodule.c 4 +Modules/mmapmodule.c 20 +Modules/resource.c 5 +Modules/_posixsubprocess.c 12 +Modules/selectmodule.c 4 +Modules/socketmodule.c 81 +Modules/termios.c 1 +Modules/syslogmodule.c 3 +Modules/_multiprocessing/posixshmem.c 1 +Modules/_multiprocessing/semaphore.c 1 +Modules/_ctypes/callbacks.c 4 +Modules/_ctypes/_ctypes.c 55 +Modules/_ctypes/callproc.c 13 +Modules/_ctypes/stgdict.c 19 +Modules/_ctypes/cfield.c 61 +Modules/_cursesmodule.c 28 +Modules/_sqlite/connection.c 5 +Modules/_sqlite/cursor.c 3 +Modules/_sqlite/module.c 2 +Modules/_ssl.c 31 +Modules/_hashopenssl.c 16 +Modules/_tkinter.c 10 +Modules/_xxtestfuzz/_xxtestfuzz.c 1 +Modules/_xxtestfuzz/fuzzer.c 13 +Modules/_testbuffer.c 28 +Modules/_testinternalcapi/test_lock.c 4 +Modules/_testinternalcapi.c 10 +Modules/_testinternalcapi/test_critical_sections.c 1 +Modules/_testcapimodule.c 3 +Modules/_testcapi/vectorcall.c 3 +Modules/_testcapi/heaptype.c 1 +Modules/_testcapi/unicode.c 2 +Modules/_testcapi/mem.c 2 +Modules/_testcapi/watchers.c 3 +Modules/_testcapi/long.c 2 +Modules/_testcapi/pyatomic.c 1 +Modules/_testcapi/bytes.c 1 +Modules/_testcapi/monitoring.c 3 +Modules/_testlimitedcapi/heaptype_relative.c 3 +Modules/_testlimitedcapi/object.c 2 +Modules/_testlimitedcapi/unicode.c 2 +Modules/clinic/_testclinic.c.h 1 +Modules/_testclinic.c 1 +Modules/_testmultiphase.c 1 +Modules/pyexpat.c 14 +Modules/_elementtree.c 43 +Modules/getpath.c 7 +Modules/_sre/sre.c 16 +Modules/_sre/sre_lib.h 62 From 4081a962e957cb8c2bbe7b51d6b7b364d1831b16 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 02:02:39 -0500 Subject: [PATCH 052/115] Add files to ignore --- Tools/build/.warningignore_ubuntu | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 04e0893b069d5b..c119d3e28f998e 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -236,3 +236,6 @@ Modules/_elementtree.c 43 Modules/getpath.c 7 Modules/_sre/sre.c 16 Modules/_sre/sre_lib.h 62 +Modules/_testexternalinspection.c 6 +Modules/_ctypes/_ctypes_test_generated.c.h 363 +Modules/_ctypes/_ctypes_test.c 7 From 26b40d3727b5bab6c88a37838b7d80f038f4a8a8 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 11:54:43 -0500 Subject: [PATCH 053/115] Update function signatures and make files with expected warnings list a true set --- Tools/build/check_warnings.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 2052b42d529672..3cedd92c411314 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -115,7 +115,7 @@ def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: def get_unexpected_warnings( files_with_expected_warnings: set[tuple[str, int]], - files_with_warnings: dict[str, list[dict]], + files_with_warnings: set[FileWarnings], ) -> int: """ Returns failure status if warnings discovered in list of warnings @@ -145,7 +145,7 @@ def get_unexpected_warnings( def get_unexpected_improvements( files_with_expected_warnings: set[tuple[str, int]], - files_with_warnings: dict[str, list[dict]], + files_with_warnings: set[FileWarnings], ) -> int: """ Returns failure status if there are no warnings in the list of warnings @@ -155,8 +155,7 @@ def get_unexpected_improvements( for file in files_with_expected_warnings: if file.name not in files_with_warnings.keys(): unexpected_improvements.append(file) - else: - if len(files_with_warnings[file.name]) < file.count: + elif len(files_with_warnings[file.name]) < file.count: unexpected_improvements.append(file) if unexpected_improvements: @@ -239,11 +238,11 @@ def main(argv: list[str] | None = None) -> int: # Files with expected warnings are stored as a set of tuples # where the first element is the file name and the second element # is the number of warnings expected in that file - files_with_expected_warnings = [ + files_with_expected_warnings = { FileWarnings(file.strip().split()[0], int(file.strip().split()[1])) for file in clean_files if file.strip() and not file.startswith("#") - ] + } with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f: compiler_output_file_contents = f.read() From 78b90be839423c4d0b101eb59350025027c2f273 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 12:42:46 -0500 Subject: [PATCH 054/115] Remove warnigns to ignore --- Tools/build/.warningignore_ubuntu | 236 ------------------------------ 1 file changed, 236 deletions(-) diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index c119d3e28f998e..469c727abfb11c 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -3,239 +3,3 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) -Include/internal/pycore_gc.h 1 -Include/internal/pycore_backoff.h 3 -Include/internal/pycore_list.h 1 -Include/internal/mimalloc/mimalloc/internal.h 4 -Parser/pegen.c 8 -Parser/string_parser.c 7 -Parser/parser.c 116 -Parser/action_helpers.c 3 -Parser/lexer/buffer.c 1 -Parser/lexer/lexer.c 14 -Parser/tokenizer/readline_tokenizer.c 4 -Parser/tokenizer/file_tokenizer.c 9 -Parser/tokenizer/helpers.c 8 -Include/internal/pycore_object.h 3 -Include/internal/pycore_long.h 3 -Objects/abstract.c 6 -Objects/bytes_methods.c 9 -Objects/stringlib/fastsearch.h 8 -Objects/bytearrayobject.c 46 -Objects/stringlib/join.h 4 -Objects/stringlib/transmogrify.h 27 -Objects/bytesobject.c 50 -Include/internal/pycore_dict.h 2 -Objects/call.c 13 -Objects/classobject.c 4 -Objects/codeobject.c 25 -Objects/descrobject.c 2 -Objects/genobject.c 3 -Objects/fileobject.c 3 -Objects/floatobject.c 18 -Objects/frameobject.c 18 -Objects/funcobject.c 2 -Objects/listobject.c 42 -Objects/longobject.c 58 -Objects/stringlib/eq.h 1 -Objects/dictobject.c 35 -Objects/odictobject.c 7 -Objects/memoryobject.c 12 -Objects/methodobject.c 1 -Objects/moduleobject.c 4 -Objects/object.c 1 -Objects/mimalloc/alloc.c 6 -Objects/mimalloc/arena.c 6 -Objects/mimalloc/heap.c 2 -Objects/mimalloc/init.c 2 -Objects/mimalloc/options.c 6 -Objects/mimalloc/os.c 4 -Objects/mimalloc/page-queue.c 2 -Objects/mimalloc/page.c 2 -Objects/mimalloc/random.c 1 -Objects/mimalloc/segment.c 11 -Objects/mimalloc/stats.c 5 -Objects/mimalloc/prim/unix/prim.c 6 -Objects/obmalloc.c 7 -Objects/rangeobject.c 10 -Objects/setobject.c 13 -Objects/sliceobject.c 4 -Objects/structseq.c 13 -Objects/tupleobject.c 8 -Objects/typeobject.c 42 -Objects/unicodeobject.c 155 -Objects/stringlib/replace.h 5 -Objects/stringlib/repr.h 21 -Objects/stringlib/codecs.h 12 -Objects/unicodectype.c 7 -Python/Python-ast.c 14 -Python/asdl.c 3 -Python/assemble.c 11 -Python/ast_opt.c 5 -Python/bltinmodule.c 8 -Python/ceval.c 8 -Python/generated_cases.c.h 23 -Python/codecs.c 28 -Python/compile.c 8 -Python/context.c 1 -Python/crossinterp_data_lookup.h 1 -Python/crossinterp.c 2 -Python/errors.c 1 -Python/flowgraph.c 8 -Python/frame.c 3 -Python/gc.c 8 -Python/getargs.c 13 -Python/getversion.c 1 -Python/ceval_gil.c 2 -Python/hashtable.c 1 -Python/import.c 6 -Python/initconfig.c 3 -Python/instrumentation.c 42 -Python/intrinsics.c 1 -Python/lock.c 4 -Python/legacy_tracing.c 3 -Python/marshal.c 17 -Python/modsupport.c 3 -Python/mystrtoul.c 4 -Python/pathconfig.c 1 -Python/pyarena.c 1 -Python/preconfig.c 2 -Python/pyhash.c 4 -Python/pylifecycle.c 7 -Python/pystate.c 6 -Python/pytime.c 2 -Python/qsbr.c 2 -Python/bootstrap_hash.c 7 -Python/specialize.c 7 -Python/symtable.c 18 -Python/sysmodule.c 3 -Python/thread_pthread.h 6 -Python/thread.c 1 -Python/traceback.c 6 -Python/tracemalloc.c 6 -Python/pystrtod.c 14 -Python/pystrhex.c 18 -Python/dtoa.c 32 -Python/formatter_unicode.c 6 -Python/fileutils.c 11 -Python/suggestions.c 12 -Python/perf_trampoline.c 12 -Python/perf_jit_trampoline.c 32 -Modules/main.c 2 -Modules/atexitmodule.c 1 -Modules/faulthandler.c 5 -Modules/signalmodule.c 3 -Modules/posixmodule.c 89 -Modules/_collectionsmodule.c 2 -Modules/_io/_iomodule.c 1 -Modules/_io/iobase.c 1 -Modules/_io/fileio.c 9 -Modules/_io/bytesio.c 15 -Modules/_io/bufferedio.c 16 -Modules/_io/textio.c 17 -Modules/_io/stringio.c 8 -Modules/itertoolsmodule.c 9 -Modules/timemodule.c 10 -Modules/_functoolsmodule.c 6 -Modules/_localemodule.c 3 -Modules/_operator.c 5 -Modules/pwdmodule.c 4 -Programs/_freeze_module.c 1 -Modules/clinic/arraymodule.c.h 1 -Modules/arraymodule.c 50 -Modules/_asynciomodule.c 3 -Modules/_bisectmodule.c 4 -Modules/_csv.c 3 -Modules/_json.c 19 -Modules/_lsprof.c 5 -Modules/rotatingtree.c 2 -Modules/_pickle.c 75 -Modules/_queuemodule.c 4 -Modules/_randommodule.c 3 -Modules/_struct.c 4 -Modules/_interpqueuesmodule.c 1 -Modules/_interpchannelsmodule.c 1 -Modules/_zoneinfo.c 17 -Modules/cmathmodule.c 1 -Modules/mathmodule.c 15 -Modules/_datetimemodule.c 34 -Include/internal/pycore_blocks_output_buffer.h 2 -Modules/_bz2module.c 5 -Modules/_decimal/_decimal.c 19 -Modules/binascii.c 208 -Modules/zlibmodule.c 26 -Modules/_lzmamodule.c 6 -Modules/_dbmmodule.c 8 -Modules/_gdbmmodule.c 5 -Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4 -Modules/readline.c 1 -Modules/sha3module.c 4 -Modules/_blake2/blake2b_impl.c 12 -Modules/_blake2/blake2s_impl.c 12 -Modules/expat/siphash.h 1 -Modules/expat/xmlparse.c 46 -Modules/expat/xmltok.c 13 -Modules/expat/xmltok_impl.c 8 -Modules/cjkcodecs/_codecs_cn.c 11 -Modules/cjkcodecs/_codecs_hk.c 2 -Modules/cjkcodecs/_codecs_iso2022.c 10 -Modules/cjkcodecs/_codecs_jp.c 32 -Modules/cjkcodecs/_codecs_kr.c 15 -Modules/cjkcodecs/_codecs_tw.c 4 -Modules/cjkcodecs/multibytecodec.c 13 -Modules/fcntlmodule.c 6 -Modules/unicodedata_db.h 1 -Modules/unicodedata.c 28 -Modules/clinic/unicodedata.c.h 10 -Modules/grpmodule.c 4 -Modules/mmapmodule.c 20 -Modules/resource.c 5 -Modules/_posixsubprocess.c 12 -Modules/selectmodule.c 4 -Modules/socketmodule.c 81 -Modules/termios.c 1 -Modules/syslogmodule.c 3 -Modules/_multiprocessing/posixshmem.c 1 -Modules/_multiprocessing/semaphore.c 1 -Modules/_ctypes/callbacks.c 4 -Modules/_ctypes/_ctypes.c 55 -Modules/_ctypes/callproc.c 13 -Modules/_ctypes/stgdict.c 19 -Modules/_ctypes/cfield.c 61 -Modules/_cursesmodule.c 28 -Modules/_sqlite/connection.c 5 -Modules/_sqlite/cursor.c 3 -Modules/_sqlite/module.c 2 -Modules/_ssl.c 31 -Modules/_hashopenssl.c 16 -Modules/_tkinter.c 10 -Modules/_xxtestfuzz/_xxtestfuzz.c 1 -Modules/_xxtestfuzz/fuzzer.c 13 -Modules/_testbuffer.c 28 -Modules/_testinternalcapi/test_lock.c 4 -Modules/_testinternalcapi.c 10 -Modules/_testinternalcapi/test_critical_sections.c 1 -Modules/_testcapimodule.c 3 -Modules/_testcapi/vectorcall.c 3 -Modules/_testcapi/heaptype.c 1 -Modules/_testcapi/unicode.c 2 -Modules/_testcapi/mem.c 2 -Modules/_testcapi/watchers.c 3 -Modules/_testcapi/long.c 2 -Modules/_testcapi/pyatomic.c 1 -Modules/_testcapi/bytes.c 1 -Modules/_testcapi/monitoring.c 3 -Modules/_testlimitedcapi/heaptype_relative.c 3 -Modules/_testlimitedcapi/object.c 2 -Modules/_testlimitedcapi/unicode.c 2 -Modules/clinic/_testclinic.c.h 1 -Modules/_testclinic.c 1 -Modules/_testmultiphase.c 1 -Modules/pyexpat.c 14 -Modules/_elementtree.c 43 -Modules/getpath.c 7 -Modules/_sre/sre.c 16 -Modules/_sre/sre_lib.h 62 -Modules/_testexternalinspection.c 6 -Modules/_ctypes/_ctypes_test_generated.c.h 363 -Modules/_ctypes/_ctypes_test.c 7 From c5a983bb2e4f64687bf4206eb97fbdebd781a964 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 13:19:02 -0500 Subject: [PATCH 055/115] Add warnings to ignore --- Tools/build/.warningignore_ubuntu | 236 ++++++++++++++++++++++++++++++ Tools/build/check_warnings.py | 3 +- 2 files changed, 238 insertions(+), 1 deletion(-) diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 469c727abfb11c..06a6cd33aa1343 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -3,3 +3,239 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) +Include/internal/pycore_gc.h 1 +Include/internal/pycore_backoff.h 3 +Include/internal/pycore_list.h 1 +Include/internal/mimalloc/mimalloc/internal.h 4 +Parser/pegen.c 8 +Parser/string_parser.c 7 +Parser/action_helpers.c 3 +Parser/lexer/buffer.c 1 +Parser/lexer/lexer.c 14 +Parser/tokenizer/file_tokenizer.c 9 +Parser/tokenizer/readline_tokenizer.c 4 +Parser/tokenizer/helpers.c 8 +Include/internal/pycore_long.h 3 +Include/internal/pycore_object.h 3 +Objects/bytes_methods.c 9 +Objects/stringlib/fastsearch.h 8 +Objects/abstract.c 6 +Include/internal/pycore_dict.h 2 +Objects/call.c 13 +Objects/bytearrayobject.c 46 +Objects/stringlib/join.h 4 +Objects/stringlib/transmogrify.h 27 +Objects/bytesobject.c 50 +Objects/classobject.c 4 +Objects/codeobject.c 25 +Objects/descrobject.c 2 +Objects/fileobject.c 3 +Objects/genobject.c 3 +Objects/floatobject.c 18 +Objects/frameobject.c 18 +Objects/funcobject.c 2 +Parser/parser.c 116 +Objects/odictobject.c 7 +Objects/listobject.c 42 +Objects/methodobject.c 1 +Objects/longobject.c 58 +Objects/memoryobject.c 12 +Objects/stringlib/eq.h 1 +Objects/dictobject.c 35 +Objects/moduleobject.c 4 +Objects/object.c 1 +Objects/rangeobject.c 10 +Objects/sliceobject.c 4 +Objects/setobject.c 13 +Objects/structseq.c 13 +Objects/tupleobject.c 8 +Objects/unicodectype.c 7 +Objects/mimalloc/alloc.c 6 +Objects/mimalloc/arena.c 6 +Objects/mimalloc/heap.c 2 +Objects/mimalloc/init.c 2 +Objects/mimalloc/options.c 6 +Objects/mimalloc/os.c 4 +Objects/mimalloc/page-queue.c 2 +Objects/mimalloc/page.c 2 +Objects/mimalloc/random.c 1 +Objects/mimalloc/segment.c 11 +Objects/mimalloc/stats.c 5 +Objects/mimalloc/prim/unix/prim.c 6 +Objects/obmalloc.c 7 +Objects/typeobject.c 42 +Python/asdl.c 3 +Python/assemble.c 11 +Python/ast_opt.c 5 +Python/bltinmodule.c 8 +Python/codecs.c 28 +Objects/unicodeobject.c 155 +Objects/stringlib/replace.h 5 +Objects/stringlib/repr.h 21 +Objects/stringlib/codecs.h 12 +Python/compile.c 8 +Python/context.c 1 +Python/Python-ast.c 14 +Python/crossinterp_data_lookup.h 1 +Python/crossinterp.c 2 +Python/ceval.c 8 +Python/generated_cases.c.h 23 +Python/errors.c 1 +Python/flowgraph.c 8 +Python/frame.c 3 +Python/gc.c 8 +Python/getversion.c 1 +Python/getargs.c 13 +Python/hashtable.c 1 +Python/ceval_gil.c 2 +Python/initconfig.c 3 +Python/import.c 6 +Python/intrinsics.c 1 +Python/instrumentation.c 42 +Python/lock.c 4 +Python/legacy_tracing.c 3 +Python/modsupport.c 3 +Python/mystrtoul.c 4 +Python/marshal.c 17 +Python/pathconfig.c 1 +Python/pyarena.c 1 +Python/preconfig.c 2 +Python/pyhash.c 4 +Python/pytime.c 2 +Python/qsbr.c 2 +Python/pystate.c 6 +Python/bootstrap_hash.c 7 +Python/specialize.c 7 +Python/pylifecycle.c 7 +Python/thread_pthread.h 6 +Python/thread.c 1 +Python/symtable.c 18 +Python/traceback.c 6 +Python/sysmodule.c 3 +Python/tracemalloc.c 6 +Python/pystrtod.c 14 +Python/pystrhex.c 18 +Python/suggestions.c 12 +Python/dtoa.c 32 +Python/formatter_unicode.c 6 +Python/fileutils.c 11 +Python/perf_trampoline.c 12 +Python/perf_jit_trampoline.c 32 +Modules/atexitmodule.c 1 +Modules/main.c 2 +Modules/faulthandler.c 5 +Modules/signalmodule.c 3 +Modules/_collectionsmodule.c 2 +Modules/_io/_iomodule.c 1 +Modules/_io/iobase.c 1 +Modules/_io/fileio.c 9 +Modules/_io/bytesio.c 15 +Modules/_io/stringio.c 8 +Modules/_io/bufferedio.c 16 +Modules/posixmodule.c 89 +Modules/_io/textio.c 17 +Modules/timemodule.c 10 +Modules/itertoolsmodule.c 9 +Modules/_sre/sre.c 16 +Modules/_sre/sre_lib.h 62 +Modules/_localemodule.c 3 +Modules/_functoolsmodule.c 6 +Modules/pwdmodule.c 4 +Programs/_freeze_module.c 1 +Modules/_operator.c 5 +Modules/_bisectmodule.c 4 +Modules/clinic/arraymodule.c.h 1 +Modules/arraymodule.c 50 +Modules/_csv.c 3 +Modules/rotatingtree.c 2 +Modules/_asynciomodule.c 3 +Modules/_lsprof.c 5 +Modules/_json.c 19 +Modules/_queuemodule.c 4 +Modules/_randommodule.c 3 +Modules/_struct.c 4 +Modules/_interpchannelsmodule.c 1 +Modules/_interpqueuesmodule.c 1 +Modules/_pickle.c 75 +Modules/_zoneinfo.c 17 +Modules/mathmodule.c 15 +Modules/cmathmodule.c 1 +Modules/binascii.c 208 +Include/internal/pycore_blocks_output_buffer.h 2 +Modules/_bz2module.c 5 +Modules/_datetimemodule.c 34 +Modules/_lzmamodule.c 6 +Modules/zlibmodule.c 26 +Modules/_dbmmodule.c 8 +Modules/_decimal/_decimal.c 19 +Modules/_gdbmmodule.c 5 +Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4 +Modules/readline.c 1 +Modules/sha3module.c 4 +Modules/_blake2/blake2b_impl.c 12 +Modules/_blake2/blake2s_impl.c 12 +Modules/cjkcodecs/_codecs_cn.c 11 +Modules/expat/siphash.h 1 +Modules/expat/xmlparse.c 46 +Modules/cjkcodecs/_codecs_hk.c 2 +Modules/cjkcodecs/_codecs_iso2022.c 10 +Modules/expat/xmltok.c 13 +Modules/expat/xmltok_impl.c 8 +Modules/cjkcodecs/_codecs_kr.c 15 +Modules/cjkcodecs/_codecs_tw.c 4 +Modules/cjkcodecs/_codecs_jp.c 32 +Modules/fcntlmodule.c 6 +Modules/cjkcodecs/multibytecodec.c 13 +Modules/grpmodule.c 4 +Modules/resource.c 5 +Modules/mmapmodule.c 20 +Modules/_posixsubprocess.c 12 +Modules/selectmodule.c 4 +Modules/unicodedata_db.h 1 +Modules/unicodedata.c 28 +Modules/clinic/unicodedata.c.h 10 +Modules/syslogmodule.c 3 +Modules/_multiprocessing/posixshmem.c 1 +Modules/termios.c 1 +Modules/_multiprocessing/semaphore.c 1 +Modules/socketmodule.c 81 +Modules/_ctypes/callbacks.c 4 +Modules/_ctypes/callproc.c 13 +Modules/_ctypes/stgdict.c 19 +Modules/_ctypes/cfield.c 61 +Modules/_ctypes/_ctypes.c 55 +Modules/_sqlite/cursor.c 3 +Modules/_sqlite/module.c 2 +Modules/_sqlite/connection.c 5 +Modules/_cursesmodule.c 28 +Modules/_hashopenssl.c 16 +Modules/_xxtestfuzz/_xxtestfuzz.c 1 +Modules/_xxtestfuzz/fuzzer.c 13 +Modules/_tkinter.c 10 +Modules/_ssl.c 31 +Modules/_testinternalcapi/test_lock.c 4 +Modules/_testbuffer.c 28 +Modules/_testinternalcapi.c 10 +Modules/_testinternalcapi/test_critical_sections.c 1 +Modules/_testcapi/vectorcall.c 3 +Modules/_testcapi/heaptype.c 1 +Modules/_testcapi/unicode.c 2 +Modules/_testcapimodule.c 3 +Modules/_testcapi/mem.c 2 +Modules/_testcapi/long.c 2 +Modules/_testcapi/watchers.c 3 +Modules/_testcapi/pyatomic.c 1 +Modules/_testcapi/bytes.c 1 +Modules/_testcapi/monitoring.c 3 +Modules/_testlimitedcapi/heaptype_relative.c 3 +Modules/_testlimitedcapi/object.c 2 +Modules/_testmultiphase.c 1 +Modules/_testexternalinspection.c 6 +Modules/_testlimitedcapi/unicode.c 2 +Modules/clinic/_testclinic.c.h 1 +Modules/_testclinic.c 1 +Modules/_ctypes/_ctypes_test_generated.c.h 363 +Modules/_ctypes/_ctypes_test.c 7 +Modules/getpath.c 7 +Modules/pyexpat.c 14 +Modules/_elementtree.c 43 diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index a48e49ebbab641..87a6ed328dfa96 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -155,9 +155,11 @@ def get_unexpected_improvements( unexpected_improvements = [] for file in files_with_expected_warnings: if file.name not in files_with_warnings.keys(): + import pdb; pdb.set_trace() unexpected_improvements.append(file) else: if len(files_with_warnings[file.name]) < file.count: + import pdb; pdb.set_trace() unexpected_improvements.append(file) if unexpected_improvements: @@ -263,7 +265,6 @@ def main(argv: list[str] | None = None) -> int: elif args.compiler_output_type == "clang": warnings = extract_warnings_from_compiler_output_clang( compiler_output_file_contents, - args.path_prefix ) files_with_warnings = get_warnings_by_file(warnings) From e8b57dca182c2754bb2da2f347df0f560e06e79e Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 13:32:36 -0500 Subject: [PATCH 056/115] Remove debug traces --- Tools/build/check_warnings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 87a6ed328dfa96..d858e4e6ba08ca 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -155,11 +155,9 @@ def get_unexpected_improvements( unexpected_improvements = [] for file in files_with_expected_warnings: if file.name not in files_with_warnings.keys(): - import pdb; pdb.set_trace() unexpected_improvements.append(file) else: if len(files_with_warnings[file.name]) < file.count: - import pdb; pdb.set_trace() unexpected_improvements.append(file) if unexpected_improvements: From 3051648ac77860ec2b0a3e34cbea9001d122e542 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 16:36:30 -0500 Subject: [PATCH 057/115] Add debug outputs --- Tools/build/check_warnings.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index d858e4e6ba08ca..69134bac85d6a4 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -265,7 +265,15 @@ def main(argv: list[str] | None = None) -> int: compiler_output_file_contents, ) + print("######### Warnings by file #########") files_with_warnings = get_warnings_by_file(warnings) + for filename in files_with_warnings.keys(): + print(f"{filename} {len(files_with_warnings[filename])}") + + print("######### Expected warnings #########") + for file in files_with_expected_warnings: + print(f"{file.name} {file.count}") + print("####################################") status = get_unexpected_warnings( files_with_expected_warnings, files_with_warnings From d351928639c1997135773825617cdc78df5a6159 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 16:42:25 -0500 Subject: [PATCH 058/115] Remove warning ignore files --- Tools/build/.warningignore_ubuntu | 236 ------------------------------ 1 file changed, 236 deletions(-) diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 06a6cd33aa1343..469c727abfb11c 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -3,239 +3,3 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) -Include/internal/pycore_gc.h 1 -Include/internal/pycore_backoff.h 3 -Include/internal/pycore_list.h 1 -Include/internal/mimalloc/mimalloc/internal.h 4 -Parser/pegen.c 8 -Parser/string_parser.c 7 -Parser/action_helpers.c 3 -Parser/lexer/buffer.c 1 -Parser/lexer/lexer.c 14 -Parser/tokenizer/file_tokenizer.c 9 -Parser/tokenizer/readline_tokenizer.c 4 -Parser/tokenizer/helpers.c 8 -Include/internal/pycore_long.h 3 -Include/internal/pycore_object.h 3 -Objects/bytes_methods.c 9 -Objects/stringlib/fastsearch.h 8 -Objects/abstract.c 6 -Include/internal/pycore_dict.h 2 -Objects/call.c 13 -Objects/bytearrayobject.c 46 -Objects/stringlib/join.h 4 -Objects/stringlib/transmogrify.h 27 -Objects/bytesobject.c 50 -Objects/classobject.c 4 -Objects/codeobject.c 25 -Objects/descrobject.c 2 -Objects/fileobject.c 3 -Objects/genobject.c 3 -Objects/floatobject.c 18 -Objects/frameobject.c 18 -Objects/funcobject.c 2 -Parser/parser.c 116 -Objects/odictobject.c 7 -Objects/listobject.c 42 -Objects/methodobject.c 1 -Objects/longobject.c 58 -Objects/memoryobject.c 12 -Objects/stringlib/eq.h 1 -Objects/dictobject.c 35 -Objects/moduleobject.c 4 -Objects/object.c 1 -Objects/rangeobject.c 10 -Objects/sliceobject.c 4 -Objects/setobject.c 13 -Objects/structseq.c 13 -Objects/tupleobject.c 8 -Objects/unicodectype.c 7 -Objects/mimalloc/alloc.c 6 -Objects/mimalloc/arena.c 6 -Objects/mimalloc/heap.c 2 -Objects/mimalloc/init.c 2 -Objects/mimalloc/options.c 6 -Objects/mimalloc/os.c 4 -Objects/mimalloc/page-queue.c 2 -Objects/mimalloc/page.c 2 -Objects/mimalloc/random.c 1 -Objects/mimalloc/segment.c 11 -Objects/mimalloc/stats.c 5 -Objects/mimalloc/prim/unix/prim.c 6 -Objects/obmalloc.c 7 -Objects/typeobject.c 42 -Python/asdl.c 3 -Python/assemble.c 11 -Python/ast_opt.c 5 -Python/bltinmodule.c 8 -Python/codecs.c 28 -Objects/unicodeobject.c 155 -Objects/stringlib/replace.h 5 -Objects/stringlib/repr.h 21 -Objects/stringlib/codecs.h 12 -Python/compile.c 8 -Python/context.c 1 -Python/Python-ast.c 14 -Python/crossinterp_data_lookup.h 1 -Python/crossinterp.c 2 -Python/ceval.c 8 -Python/generated_cases.c.h 23 -Python/errors.c 1 -Python/flowgraph.c 8 -Python/frame.c 3 -Python/gc.c 8 -Python/getversion.c 1 -Python/getargs.c 13 -Python/hashtable.c 1 -Python/ceval_gil.c 2 -Python/initconfig.c 3 -Python/import.c 6 -Python/intrinsics.c 1 -Python/instrumentation.c 42 -Python/lock.c 4 -Python/legacy_tracing.c 3 -Python/modsupport.c 3 -Python/mystrtoul.c 4 -Python/marshal.c 17 -Python/pathconfig.c 1 -Python/pyarena.c 1 -Python/preconfig.c 2 -Python/pyhash.c 4 -Python/pytime.c 2 -Python/qsbr.c 2 -Python/pystate.c 6 -Python/bootstrap_hash.c 7 -Python/specialize.c 7 -Python/pylifecycle.c 7 -Python/thread_pthread.h 6 -Python/thread.c 1 -Python/symtable.c 18 -Python/traceback.c 6 -Python/sysmodule.c 3 -Python/tracemalloc.c 6 -Python/pystrtod.c 14 -Python/pystrhex.c 18 -Python/suggestions.c 12 -Python/dtoa.c 32 -Python/formatter_unicode.c 6 -Python/fileutils.c 11 -Python/perf_trampoline.c 12 -Python/perf_jit_trampoline.c 32 -Modules/atexitmodule.c 1 -Modules/main.c 2 -Modules/faulthandler.c 5 -Modules/signalmodule.c 3 -Modules/_collectionsmodule.c 2 -Modules/_io/_iomodule.c 1 -Modules/_io/iobase.c 1 -Modules/_io/fileio.c 9 -Modules/_io/bytesio.c 15 -Modules/_io/stringio.c 8 -Modules/_io/bufferedio.c 16 -Modules/posixmodule.c 89 -Modules/_io/textio.c 17 -Modules/timemodule.c 10 -Modules/itertoolsmodule.c 9 -Modules/_sre/sre.c 16 -Modules/_sre/sre_lib.h 62 -Modules/_localemodule.c 3 -Modules/_functoolsmodule.c 6 -Modules/pwdmodule.c 4 -Programs/_freeze_module.c 1 -Modules/_operator.c 5 -Modules/_bisectmodule.c 4 -Modules/clinic/arraymodule.c.h 1 -Modules/arraymodule.c 50 -Modules/_csv.c 3 -Modules/rotatingtree.c 2 -Modules/_asynciomodule.c 3 -Modules/_lsprof.c 5 -Modules/_json.c 19 -Modules/_queuemodule.c 4 -Modules/_randommodule.c 3 -Modules/_struct.c 4 -Modules/_interpchannelsmodule.c 1 -Modules/_interpqueuesmodule.c 1 -Modules/_pickle.c 75 -Modules/_zoneinfo.c 17 -Modules/mathmodule.c 15 -Modules/cmathmodule.c 1 -Modules/binascii.c 208 -Include/internal/pycore_blocks_output_buffer.h 2 -Modules/_bz2module.c 5 -Modules/_datetimemodule.c 34 -Modules/_lzmamodule.c 6 -Modules/zlibmodule.c 26 -Modules/_dbmmodule.c 8 -Modules/_decimal/_decimal.c 19 -Modules/_gdbmmodule.c 5 -Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4 -Modules/readline.c 1 -Modules/sha3module.c 4 -Modules/_blake2/blake2b_impl.c 12 -Modules/_blake2/blake2s_impl.c 12 -Modules/cjkcodecs/_codecs_cn.c 11 -Modules/expat/siphash.h 1 -Modules/expat/xmlparse.c 46 -Modules/cjkcodecs/_codecs_hk.c 2 -Modules/cjkcodecs/_codecs_iso2022.c 10 -Modules/expat/xmltok.c 13 -Modules/expat/xmltok_impl.c 8 -Modules/cjkcodecs/_codecs_kr.c 15 -Modules/cjkcodecs/_codecs_tw.c 4 -Modules/cjkcodecs/_codecs_jp.c 32 -Modules/fcntlmodule.c 6 -Modules/cjkcodecs/multibytecodec.c 13 -Modules/grpmodule.c 4 -Modules/resource.c 5 -Modules/mmapmodule.c 20 -Modules/_posixsubprocess.c 12 -Modules/selectmodule.c 4 -Modules/unicodedata_db.h 1 -Modules/unicodedata.c 28 -Modules/clinic/unicodedata.c.h 10 -Modules/syslogmodule.c 3 -Modules/_multiprocessing/posixshmem.c 1 -Modules/termios.c 1 -Modules/_multiprocessing/semaphore.c 1 -Modules/socketmodule.c 81 -Modules/_ctypes/callbacks.c 4 -Modules/_ctypes/callproc.c 13 -Modules/_ctypes/stgdict.c 19 -Modules/_ctypes/cfield.c 61 -Modules/_ctypes/_ctypes.c 55 -Modules/_sqlite/cursor.c 3 -Modules/_sqlite/module.c 2 -Modules/_sqlite/connection.c 5 -Modules/_cursesmodule.c 28 -Modules/_hashopenssl.c 16 -Modules/_xxtestfuzz/_xxtestfuzz.c 1 -Modules/_xxtestfuzz/fuzzer.c 13 -Modules/_tkinter.c 10 -Modules/_ssl.c 31 -Modules/_testinternalcapi/test_lock.c 4 -Modules/_testbuffer.c 28 -Modules/_testinternalcapi.c 10 -Modules/_testinternalcapi/test_critical_sections.c 1 -Modules/_testcapi/vectorcall.c 3 -Modules/_testcapi/heaptype.c 1 -Modules/_testcapi/unicode.c 2 -Modules/_testcapimodule.c 3 -Modules/_testcapi/mem.c 2 -Modules/_testcapi/long.c 2 -Modules/_testcapi/watchers.c 3 -Modules/_testcapi/pyatomic.c 1 -Modules/_testcapi/bytes.c 1 -Modules/_testcapi/monitoring.c 3 -Modules/_testlimitedcapi/heaptype_relative.c 3 -Modules/_testlimitedcapi/object.c 2 -Modules/_testmultiphase.c 1 -Modules/_testexternalinspection.c 6 -Modules/_testlimitedcapi/unicode.c 2 -Modules/clinic/_testclinic.c.h 1 -Modules/_testclinic.c 1 -Modules/_ctypes/_ctypes_test_generated.c.h 363 -Modules/_ctypes/_ctypes_test.c 7 -Modules/getpath.c 7 -Modules/pyexpat.c 14 -Modules/_elementtree.c 43 From 69f37eaad29a217f7ee3b9656fbe9ebb28ab1abb Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 17:53:41 -0500 Subject: [PATCH 059/115] Make warning check only run on non-free-threading jobs --- .github/workflows/reusable-ubuntu.yml | 1 + Tools/build/.warningignore_ubuntu | 236 ++++++++++++++++++++++++++ 2 files changed, 237 insertions(+) diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 766b15d03c49df..931b0678eef243 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -80,6 +80,7 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo - name: Check compiler warnings + if: ${{ !fromJSON(inputs.free-threading) }} run: >- python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 469c727abfb11c..06a6cd33aa1343 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -3,3 +3,239 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) +Include/internal/pycore_gc.h 1 +Include/internal/pycore_backoff.h 3 +Include/internal/pycore_list.h 1 +Include/internal/mimalloc/mimalloc/internal.h 4 +Parser/pegen.c 8 +Parser/string_parser.c 7 +Parser/action_helpers.c 3 +Parser/lexer/buffer.c 1 +Parser/lexer/lexer.c 14 +Parser/tokenizer/file_tokenizer.c 9 +Parser/tokenizer/readline_tokenizer.c 4 +Parser/tokenizer/helpers.c 8 +Include/internal/pycore_long.h 3 +Include/internal/pycore_object.h 3 +Objects/bytes_methods.c 9 +Objects/stringlib/fastsearch.h 8 +Objects/abstract.c 6 +Include/internal/pycore_dict.h 2 +Objects/call.c 13 +Objects/bytearrayobject.c 46 +Objects/stringlib/join.h 4 +Objects/stringlib/transmogrify.h 27 +Objects/bytesobject.c 50 +Objects/classobject.c 4 +Objects/codeobject.c 25 +Objects/descrobject.c 2 +Objects/fileobject.c 3 +Objects/genobject.c 3 +Objects/floatobject.c 18 +Objects/frameobject.c 18 +Objects/funcobject.c 2 +Parser/parser.c 116 +Objects/odictobject.c 7 +Objects/listobject.c 42 +Objects/methodobject.c 1 +Objects/longobject.c 58 +Objects/memoryobject.c 12 +Objects/stringlib/eq.h 1 +Objects/dictobject.c 35 +Objects/moduleobject.c 4 +Objects/object.c 1 +Objects/rangeobject.c 10 +Objects/sliceobject.c 4 +Objects/setobject.c 13 +Objects/structseq.c 13 +Objects/tupleobject.c 8 +Objects/unicodectype.c 7 +Objects/mimalloc/alloc.c 6 +Objects/mimalloc/arena.c 6 +Objects/mimalloc/heap.c 2 +Objects/mimalloc/init.c 2 +Objects/mimalloc/options.c 6 +Objects/mimalloc/os.c 4 +Objects/mimalloc/page-queue.c 2 +Objects/mimalloc/page.c 2 +Objects/mimalloc/random.c 1 +Objects/mimalloc/segment.c 11 +Objects/mimalloc/stats.c 5 +Objects/mimalloc/prim/unix/prim.c 6 +Objects/obmalloc.c 7 +Objects/typeobject.c 42 +Python/asdl.c 3 +Python/assemble.c 11 +Python/ast_opt.c 5 +Python/bltinmodule.c 8 +Python/codecs.c 28 +Objects/unicodeobject.c 155 +Objects/stringlib/replace.h 5 +Objects/stringlib/repr.h 21 +Objects/stringlib/codecs.h 12 +Python/compile.c 8 +Python/context.c 1 +Python/Python-ast.c 14 +Python/crossinterp_data_lookup.h 1 +Python/crossinterp.c 2 +Python/ceval.c 8 +Python/generated_cases.c.h 23 +Python/errors.c 1 +Python/flowgraph.c 8 +Python/frame.c 3 +Python/gc.c 8 +Python/getversion.c 1 +Python/getargs.c 13 +Python/hashtable.c 1 +Python/ceval_gil.c 2 +Python/initconfig.c 3 +Python/import.c 6 +Python/intrinsics.c 1 +Python/instrumentation.c 42 +Python/lock.c 4 +Python/legacy_tracing.c 3 +Python/modsupport.c 3 +Python/mystrtoul.c 4 +Python/marshal.c 17 +Python/pathconfig.c 1 +Python/pyarena.c 1 +Python/preconfig.c 2 +Python/pyhash.c 4 +Python/pytime.c 2 +Python/qsbr.c 2 +Python/pystate.c 6 +Python/bootstrap_hash.c 7 +Python/specialize.c 7 +Python/pylifecycle.c 7 +Python/thread_pthread.h 6 +Python/thread.c 1 +Python/symtable.c 18 +Python/traceback.c 6 +Python/sysmodule.c 3 +Python/tracemalloc.c 6 +Python/pystrtod.c 14 +Python/pystrhex.c 18 +Python/suggestions.c 12 +Python/dtoa.c 32 +Python/formatter_unicode.c 6 +Python/fileutils.c 11 +Python/perf_trampoline.c 12 +Python/perf_jit_trampoline.c 32 +Modules/atexitmodule.c 1 +Modules/main.c 2 +Modules/faulthandler.c 5 +Modules/signalmodule.c 3 +Modules/_collectionsmodule.c 2 +Modules/_io/_iomodule.c 1 +Modules/_io/iobase.c 1 +Modules/_io/fileio.c 9 +Modules/_io/bytesio.c 15 +Modules/_io/stringio.c 8 +Modules/_io/bufferedio.c 16 +Modules/posixmodule.c 89 +Modules/_io/textio.c 17 +Modules/timemodule.c 10 +Modules/itertoolsmodule.c 9 +Modules/_sre/sre.c 16 +Modules/_sre/sre_lib.h 62 +Modules/_localemodule.c 3 +Modules/_functoolsmodule.c 6 +Modules/pwdmodule.c 4 +Programs/_freeze_module.c 1 +Modules/_operator.c 5 +Modules/_bisectmodule.c 4 +Modules/clinic/arraymodule.c.h 1 +Modules/arraymodule.c 50 +Modules/_csv.c 3 +Modules/rotatingtree.c 2 +Modules/_asynciomodule.c 3 +Modules/_lsprof.c 5 +Modules/_json.c 19 +Modules/_queuemodule.c 4 +Modules/_randommodule.c 3 +Modules/_struct.c 4 +Modules/_interpchannelsmodule.c 1 +Modules/_interpqueuesmodule.c 1 +Modules/_pickle.c 75 +Modules/_zoneinfo.c 17 +Modules/mathmodule.c 15 +Modules/cmathmodule.c 1 +Modules/binascii.c 208 +Include/internal/pycore_blocks_output_buffer.h 2 +Modules/_bz2module.c 5 +Modules/_datetimemodule.c 34 +Modules/_lzmamodule.c 6 +Modules/zlibmodule.c 26 +Modules/_dbmmodule.c 8 +Modules/_decimal/_decimal.c 19 +Modules/_gdbmmodule.c 5 +Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4 +Modules/readline.c 1 +Modules/sha3module.c 4 +Modules/_blake2/blake2b_impl.c 12 +Modules/_blake2/blake2s_impl.c 12 +Modules/cjkcodecs/_codecs_cn.c 11 +Modules/expat/siphash.h 1 +Modules/expat/xmlparse.c 46 +Modules/cjkcodecs/_codecs_hk.c 2 +Modules/cjkcodecs/_codecs_iso2022.c 10 +Modules/expat/xmltok.c 13 +Modules/expat/xmltok_impl.c 8 +Modules/cjkcodecs/_codecs_kr.c 15 +Modules/cjkcodecs/_codecs_tw.c 4 +Modules/cjkcodecs/_codecs_jp.c 32 +Modules/fcntlmodule.c 6 +Modules/cjkcodecs/multibytecodec.c 13 +Modules/grpmodule.c 4 +Modules/resource.c 5 +Modules/mmapmodule.c 20 +Modules/_posixsubprocess.c 12 +Modules/selectmodule.c 4 +Modules/unicodedata_db.h 1 +Modules/unicodedata.c 28 +Modules/clinic/unicodedata.c.h 10 +Modules/syslogmodule.c 3 +Modules/_multiprocessing/posixshmem.c 1 +Modules/termios.c 1 +Modules/_multiprocessing/semaphore.c 1 +Modules/socketmodule.c 81 +Modules/_ctypes/callbacks.c 4 +Modules/_ctypes/callproc.c 13 +Modules/_ctypes/stgdict.c 19 +Modules/_ctypes/cfield.c 61 +Modules/_ctypes/_ctypes.c 55 +Modules/_sqlite/cursor.c 3 +Modules/_sqlite/module.c 2 +Modules/_sqlite/connection.c 5 +Modules/_cursesmodule.c 28 +Modules/_hashopenssl.c 16 +Modules/_xxtestfuzz/_xxtestfuzz.c 1 +Modules/_xxtestfuzz/fuzzer.c 13 +Modules/_tkinter.c 10 +Modules/_ssl.c 31 +Modules/_testinternalcapi/test_lock.c 4 +Modules/_testbuffer.c 28 +Modules/_testinternalcapi.c 10 +Modules/_testinternalcapi/test_critical_sections.c 1 +Modules/_testcapi/vectorcall.c 3 +Modules/_testcapi/heaptype.c 1 +Modules/_testcapi/unicode.c 2 +Modules/_testcapimodule.c 3 +Modules/_testcapi/mem.c 2 +Modules/_testcapi/long.c 2 +Modules/_testcapi/watchers.c 3 +Modules/_testcapi/pyatomic.c 1 +Modules/_testcapi/bytes.c 1 +Modules/_testcapi/monitoring.c 3 +Modules/_testlimitedcapi/heaptype_relative.c 3 +Modules/_testlimitedcapi/object.c 2 +Modules/_testmultiphase.c 1 +Modules/_testexternalinspection.c 6 +Modules/_testlimitedcapi/unicode.c 2 +Modules/clinic/_testclinic.c.h 1 +Modules/_testclinic.c 1 +Modules/_ctypes/_ctypes_test_generated.c.h 363 +Modules/_ctypes/_ctypes_test.c 7 +Modules/getpath.c 7 +Modules/pyexpat.c 14 +Modules/_elementtree.c 43 From 74164c54e57b3d1ab9af2471fac3687156607017 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Tue, 13 Aug 2024 19:09:38 -0500 Subject: [PATCH 060/115] Add option parsing for clang --- Tools/build/check_warnings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index dc3d55c3cd2090..8802a8cb736a51 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -24,7 +24,7 @@ def extract_warnings_from_compiler_output_clang( """ # Regex to find warnings in the compiler output clang_warning_regex = re.compile( - r"(?P.*):(?P\d+):(?P\d+): warning: (?P.*)" + r"(?P.*):(?P\d+):(?P\d+): warning: (?P.*) (?P