From a930d374e238ba5de6022af8141cb9edf66b2fa0 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 20 Feb 2023 10:59:30 +0300 Subject: [PATCH 1/5] gh-102033: Fix syntax error in `Tools/c-analyzer` --- Tools/c-analyzer/c_parser/parser/_delim.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/c-analyzer/c_parser/parser/_delim.py b/Tools/c-analyzer/c_parser/parser/_delim.py index 51433a629d3a35..5c45379d80c40e 100644 --- a/Tools/c-analyzer/c_parser/parser/_delim.py +++ b/Tools/c-analyzer/c_parser/parser/_delim.py @@ -17,11 +17,11 @@ def parse(text, anon_name): yield result -DELIMITER = textwrap.dedent(rf''' +DELIMITER = textwrap.dedent(r''' ( (?: [^'"()\[\]{};]* - {_ind(STRING_LITERAL, 3)} + %s }* [^'"()\[\]{};]+ )? # @@ -34,7 +34,7 @@ def parse(text, anon_name): [)\]};] ) # )? - ''') + ''' % _ind(STRING_LITERAL, 3)) DELIMITER_RE = re.compile(DELIMITER, re.VERBOSE) _HANDLERS = { From f6026e68f7ab551c2e68557a686663cbd2670af6 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 20 Feb 2023 13:54:43 +0300 Subject: [PATCH 2/5] Remove unused parts from `Tools/c-analyzer` --- Tools/c-analyzer/c_analyzer/__main__.py | 2 - Tools/c-analyzer/c_analyzer/info.py | 4 - Tools/c-analyzer/c_common/info.py | 0 Tools/c-analyzer/c_common/iterutil.py | 4 - Tools/c-analyzer/c_common/show.py | 0 Tools/c-analyzer/c_parser/__main__.py | 4 - Tools/c-analyzer/c_parser/_state_machine.py | 244 -------------------- Tools/c-analyzer/c_parser/info.py | 3 +- Tools/c-analyzer/c_parser/parser/_alt.py | 6 - Tools/c-analyzer/c_parser/parser/_delim.py | 54 ----- Tools/c-analyzer/c_parser/parser/_global.py | 1 - Tools/c-analyzer/cpython/__main__.py | 2 - Tools/c-analyzer/cpython/_analyzer.py | 4 - Tools/c-analyzer/cpython/_capi.py | 6 +- 14 files changed, 2 insertions(+), 332 deletions(-) delete mode 100644 Tools/c-analyzer/c_common/info.py delete mode 100644 Tools/c-analyzer/c_common/show.py delete mode 100644 Tools/c-analyzer/c_parser/_state_machine.py delete mode 100644 Tools/c-analyzer/c_parser/parser/_alt.py delete mode 100644 Tools/c-analyzer/c_parser/parser/_delim.py diff --git a/Tools/c-analyzer/c_analyzer/__main__.py b/Tools/c-analyzer/c_analyzer/__main__.py index 5d89b29adf899e..cde39bc4e649d9 100644 --- a/Tools/c-analyzer/c_analyzer/__main__.py +++ b/Tools/c-analyzer/c_analyzer/__main__.py @@ -18,10 +18,8 @@ configure_logger, get_prog, filter_filenames, - iter_marks, ) from c_parser.info import KIND -from c_parser.match import is_type_decl from .match import filter_forward from . import ( analyze as _analyze, diff --git a/Tools/c-analyzer/c_analyzer/info.py b/Tools/c-analyzer/c_analyzer/info.py index 27c3a5a4ee76f2..d231e07a60dd8e 100644 --- a/Tools/c-analyzer/c_analyzer/info.py +++ b/Tools/c-analyzer/c_analyzer/info.py @@ -1,4 +1,3 @@ -from collections import namedtuple import os.path from c_common import fsutil @@ -13,9 +12,6 @@ from c_parser.match import ( is_type_decl, ) -from .match import ( - is_process_global, -) IGNORED = _misc.Labeled('IGNORED') diff --git a/Tools/c-analyzer/c_common/info.py b/Tools/c-analyzer/c_common/info.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Tools/c-analyzer/c_common/iterutil.py b/Tools/c-analyzer/c_common/iterutil.py index 6ded105304e454..dda3dd57c1cf5c 100644 --- a/Tools/c-analyzer/c_common/iterutil.py +++ b/Tools/c-analyzer/c_common/iterutil.py @@ -1,7 +1,3 @@ - -_NOT_SET = object() - - def peek_and_iter(items): if not items: return None, None diff --git a/Tools/c-analyzer/c_common/show.py b/Tools/c-analyzer/c_common/show.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Tools/c-analyzer/c_parser/__main__.py b/Tools/c-analyzer/c_parser/__main__.py index 78f47a1808f50b..2454fcba814291 100644 --- a/Tools/c-analyzer/c_parser/__main__.py +++ b/Tools/c-analyzer/c_parser/__main__.py @@ -1,10 +1,7 @@ import logging -import os.path import sys -from c_common import fsutil from c_common.scriptutil import ( - CLIArgSpec as Arg, add_verbosity_cli, add_traceback_cli, add_kind_filtering_cli, @@ -15,7 +12,6 @@ get_prog, main_for_filenames, ) -from .preprocessor import get_preprocessor from .preprocessor.__main__ import ( add_common_cli as add_preprocessor_cli, ) diff --git a/Tools/c-analyzer/c_parser/_state_machine.py b/Tools/c-analyzer/c_parser/_state_machine.py deleted file mode 100644 index 875323188aadfd..00000000000000 --- a/Tools/c-analyzer/c_parser/_state_machine.py +++ /dev/null @@ -1,244 +0,0 @@ - -f''' - struct {ANON_IDENTIFIER}; - struct {{ ... }} - struct {IDENTIFIER} {{ ... }} - - union {ANON_IDENTIFIER}; - union {{ ... }} - union {IDENTIFIER} {{ ... }} - - enum {ANON_IDENTIFIER}; - enum {{ ... }} - enum {IDENTIFIER} {{ ... }} - - typedef {VARTYPE} {IDENTIFIER}; - typedef {IDENTIFIER}; - typedef {IDENTIFIER}; - typedef {IDENTIFIER}; -''' - - -def parse(srclines): - if isinstance(srclines, str): # a filename - raise NotImplementedError - - - -# This only handles at most 10 nested levels. -#MATCHED_PARENS = textwrap.dedent(rf''' -# # matched parens -# (?: -# [(] # level 0 -# (?: -# [^()]* -# [(] # level 1 -# (?: -# [^()]* -# [(] # level 2 -# (?: -# [^()]* -# [(] # level 3 -# (?: -# [^()]* -# [(] # level 4 -# (?: -# [^()]* -# [(] # level 5 -# (?: -# [^()]* -# [(] # level 6 -# (?: -# [^()]* -# [(] # level 7 -# (?: -# [^()]* -# [(] # level 8 -# (?: -# [^()]* -# [(] # level 9 -# (?: -# [^()]* -# [(] # level 10 -# [^()]* -# [)] -# )* -# [^()]* -# [)] -# )* -# [^()]* -# [)] -# )* -# [^()]* -# [)] -# )* -# [^()]* -# [)] -# )* -# [^()]* -# [)] -# )* -# [^()]* -# [)] -# )* -# [^()]* -# [)] -# )* -# [^()]* -# [)] -# )* -# [^()]* -# [)] -# )* -# [^()]* -# [)] -# ) -# # end matched parens -# ''') - -r''' - # for loop - (?: - \s* \b for - \s* [(] - ( - [^;]* ; - [^;]* ; - .*? - ) #
- [)] - \s* - (?: - (?: - ( - {_ind(SIMPLE_STMT, 6)} - ) # - ; - ) - | - ( {{ ) # - ) - ) - | - - - - ( - (?: - (?: - (?: - {_ind(SIMPLE_STMT, 6)} - )? - return \b \s* - {_ind(INITIALIZER, 5)} - ) - | - (?: - (?: - {IDENTIFIER} \s* - (?: . | -> ) \s* - )* - {IDENTIFIER} - \s* = \s* - {_ind(INITIALIZER, 5)} - ) - | - (?: - {_ind(SIMPLE_STMT, 5)} - ) - ) - | - # cast compound literal - (?: - (?: - [^'"{{}};]* - {_ind(STRING_LITERAL, 5)} - )* - [^'"{{}};]*? - [^'"{{}};=] - = - \s* [(] [^)]* [)] - \s* {{ [^;]* }} - ) - ) # - - - - # compound statement - (?: - ( - (?: - - # "for" statements are handled separately above. - (?: (?: else \s+ )? if | switch | while ) \s* - {_ind(COMPOUND_HEAD, 5)} - ) - | - (?: else | do ) - # We do not worry about compound statements for labels, - # "case", or "default". - )? #
- \s* - ( {{ ) # - ) - - - - ( - (?: - [^'"{{}};]* - {_ind(STRING_LITERAL, 5)} - )* - [^'"{{}};]* - # Presumably we will not see "== {{". - [^\s='"{{}};] - )? #
- - - - ( - \b - (?: - # We don't worry about labels with a compound statement. - (?: - switch \s* [(] [^{{]* [)] - ) - | - (?: - case \b \s* [^:]+ [:] - ) - | - (?: - default \s* [:] - ) - | - (?: - do - ) - | - (?: - while \s* [(] [^{{]* [)] - ) - | - #(?: - # for \s* [(] [^{{]* [)] - # ) - #| - (?: - if \s* [(] - (?: [^{{]* [^)] \s* {{ )* [^{{]* - [)] - ) - | - (?: - else - (?: - \s* - if \s* [(] - (?: [^{{]* [^)] \s* {{ )* [^{{]* - [)] - )? - ) - ) - )? #
-''' diff --git a/Tools/c-analyzer/c_parser/info.py b/Tools/c-analyzer/c_parser/info.py index 3fa9fefbd5ec0b..799f9237877447 100644 --- a/Tools/c-analyzer/c_parser/info.py +++ b/Tools/c-analyzer/c_parser/info.py @@ -1,6 +1,5 @@ from collections import namedtuple import enum -import os.path import re from c_common import fsutil @@ -8,7 +7,7 @@ import c_common.misc as _misc import c_common.strutil as _strutil import c_common.tables as _tables -from .parser._regexes import SIMPLE_TYPE, _STORAGE +from .parser._regexes import _STORAGE FIXED_TYPE = _misc.Labeled('FIXED_TYPE') diff --git a/Tools/c-analyzer/c_parser/parser/_alt.py b/Tools/c-analyzer/c_parser/parser/_alt.py deleted file mode 100644 index 05a9101b4f529a..00000000000000 --- a/Tools/c-analyzer/c_parser/parser/_alt.py +++ /dev/null @@ -1,6 +0,0 @@ - -def _parse(srclines, anon_name): - text = ' '.join(l for _, l in srclines) - - from ._delim import parse - yield from parse(text, anon_name) diff --git a/Tools/c-analyzer/c_parser/parser/_delim.py b/Tools/c-analyzer/c_parser/parser/_delim.py deleted file mode 100644 index 5c45379d80c40e..00000000000000 --- a/Tools/c-analyzer/c_parser/parser/_delim.py +++ /dev/null @@ -1,54 +0,0 @@ -import re -import textwrap - -from ._regexes import _ind, STRING_LITERAL - - -def parse(text, anon_name): - context = None - data = None - for m in DELIMITER_RE.find_iter(text): - before, opened, closed = m.groups() - delim = opened or closed - - handle_segment = HANDLERS[context][delim] - result, context, data = handle_segment(before, delim, data) - if result: - yield result - - -DELIMITER = textwrap.dedent(r''' - ( - (?: - [^'"()\[\]{};]* - %s - }* - [^'"()\[\]{};]+ - )? # - (?: - ( - [(\[{] - ) # - | - ( - [)\]};] - ) # - )? - ''' % _ind(STRING_LITERAL, 3)) -DELIMITER_RE = re.compile(DELIMITER, re.VERBOSE) - -_HANDLERS = { - None: { # global - # opened - '{': ..., - '[': None, - '(': None, - # closed - '}': None, - ']': None, - ')': None, - ';': ..., - }, - '': { - }, -} diff --git a/Tools/c-analyzer/c_parser/parser/_global.py b/Tools/c-analyzer/c_parser/parser/_global.py index 35947c12998135..b1ac9f5db034e1 100644 --- a/Tools/c-analyzer/c_parser/parser/_global.py +++ b/Tools/c-analyzer/c_parser/parser/_global.py @@ -9,7 +9,6 @@ set_capture_groups, ) from ._compound_decl_body import DECL_BODY_PARSERS -#from ._func_body import parse_function_body from ._func_body import parse_function_statics as parse_function_body diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py index 2b9e4233b95ac4..96d8d173e4c7e9 100644 --- a/Tools/c-analyzer/cpython/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -1,7 +1,6 @@ import logging import sys -from c_common.fsutil import expand_filenames, iter_files_by_suffix from c_common.scriptutil import ( VERBOSITY, add_verbosity_cli, @@ -10,7 +9,6 @@ add_kind_filtering_cli, add_files_cli, add_progress_cli, - main_for_filenames, process_args_by_key, configure_logger, get_prog, diff --git a/Tools/c-analyzer/cpython/_analyzer.py b/Tools/c-analyzer/cpython/_analyzer.py index cfe5e75f2f4df6..68d6b31cf2b6f0 100644 --- a/Tools/c-analyzer/cpython/_analyzer.py +++ b/Tools/c-analyzer/cpython/_analyzer.py @@ -4,16 +4,12 @@ from c_common.clsutil import classonly from c_parser.info import ( KIND, - DeclID, Declaration, TypeDeclaration, - TypeDef, - Struct, Member, FIXED_TYPE, ) from c_parser.match import ( - is_type_decl, is_pots, is_funcptr, ) diff --git a/Tools/c-analyzer/cpython/_capi.py b/Tools/c-analyzer/cpython/_capi.py index df8159a8cc169f..4b4f625651c95e 100644 --- a/Tools/c-analyzer/cpython/_capi.py +++ b/Tools/c-analyzer/cpython/_capi.py @@ -7,7 +7,7 @@ from c_common.tables import build_table, resolve_columns from c_parser.parser._regexes import _ind -from ._files import iter_header_files, resolve_filename +from ._files import iter_header_files from . import REPO_ROOT @@ -636,7 +636,6 @@ def render_summary(items, *, subtotals = summary['totals']['subs'] bygroup = summary['totals']['bygroup'] - lastempty = False for outer, subtotal in subtotals.items(): if bygroup: subtotal = f'({subtotal})' @@ -646,9 +645,6 @@ def render_summary(items, *, if outer in bygroup: for inner, count in bygroup[outer].items(): yield f' {inner + ":":9} {count}' - lastempty = False - else: - lastempty = True total = f'*{summary["totals"]["all"]}*' label = '*total*:' From 3dc3bd99d3475033350ce208fdfe20e02101ddd3 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 20 Feb 2023 13:56:39 +0300 Subject: [PATCH 3/5] Trigger a warning --- Modules/_testcapimodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 0d8d1d73fb2390..00518bd56b9f37 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3197,7 +3197,7 @@ eval_eval_code_ex(PyObject *mod, PyObject *pos_args) globals, locals, c_args, - (int)c_args_len, + c_args_len, c_kwargs, (int)c_kwargs_len, c_defaults, From a1906bc317ed09b80411caaab97c833186dc10b2 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 20 Feb 2023 14:30:26 +0300 Subject: [PATCH 4/5] Remove extra warning --- Modules/_testcapimodule.c | 2 +- Tools/c-analyzer/cpython/_capi.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 00518bd56b9f37..0d8d1d73fb2390 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3197,7 +3197,7 @@ eval_eval_code_ex(PyObject *mod, PyObject *pos_args) globals, locals, c_args, - c_args_len, + (int)c_args_len, c_kwargs, (int)c_kwargs_len, c_defaults, diff --git a/Tools/c-analyzer/cpython/_capi.py b/Tools/c-analyzer/cpython/_capi.py index 4b4f625651c95e..5584677daee2a6 100644 --- a/Tools/c-analyzer/cpython/_capi.py +++ b/Tools/c-analyzer/cpython/_capi.py @@ -645,7 +645,6 @@ def render_summary(items, *, if outer in bygroup: for inner, count in bygroup[outer].items(): yield f' {inner + ":":9} {count}' - total = f'*{summary["totals"]["all"]}*' label = '*total*:' if bygroup: From be5067270eab5f08f526d0b129c203af8e5255fa Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 20 Feb 2023 22:44:24 +0300 Subject: [PATCH 5/5] Update _capi.py --- Tools/c-analyzer/cpython/_capi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tools/c-analyzer/cpython/_capi.py b/Tools/c-analyzer/cpython/_capi.py index 5584677daee2a6..4552f71479bd06 100644 --- a/Tools/c-analyzer/cpython/_capi.py +++ b/Tools/c-analyzer/cpython/_capi.py @@ -610,8 +610,7 @@ def _render_item_full(item, groupby, verbose): yield item.name yield f' {"filename:":10} {item.relfile}' for extra in ('kind', 'level'): - #if groupby != extra: - yield f' {extra+":":10} {getattr(item, extra)}' + yield f' {extra+":":10} {getattr(item, extra)}' if verbose: print(' ---------------------------------------') for lno, line in enumerate(item.text, item.lno):