From 1d04a76d03ee7d5cda20530e0c411c3d610cee11 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 15 May 2025 12:13:03 +0300 Subject: [PATCH] [3.13] gh-133403: Check `Tools/build/deepfreeze.py` with mypy (GH-133802) (cherry picked from commit 7eaa09739059aaac4812395f8d6bb586af8eadcc) Co-authored-by: sobolevn --- .github/workflows/mypy.yml | 2 ++ Tools/build/deepfreeze.py | 54 ++++++++++++++++++++++++-------------- Tools/build/mypy.ini | 4 ++- Tools/build/umarshal.py | 11 ++++---- 4 files changed, 45 insertions(+), 26 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index cb5349bb7d4dc0..3f80f7fbdfa970 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -14,9 +14,11 @@ on: - "Lib/tomllib/**" - "Misc/mypy/**" - "Tools/build/compute-changes.py" + - "Tools/build/deepfreeze.py" - "Tools/build/generate_sbom.py" - "Tools/build/verify_ensurepip_wheels.py" - "Tools/build/update_file.py" + - "Tools/build/umarshal.py" - "Tools/cases_generator/**" - "Tools/clinic/**" - "Tools/jit/**" diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index 05633e3f77af49..35decd97d33d59 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -2,9 +2,12 @@ The script may be executed by _bootstrap_python interpreter. Shared library extension modules are not available in that case. -On Windows, and in cross-compilation cases, it is executed -by Python 3.10, and 3.11 features are not available. +Requires 3.11+ to be executed, +because relies on `code.co_qualname` and `code.co_exceptiontable`. """ + +from __future__ import annotations + import argparse import builtins import collections @@ -13,10 +16,13 @@ import re import time import types -from typing import Dict, FrozenSet, TextIO, Tuple - import umarshal +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import Any, TextIO, Dict, FrozenSet, TextIO, Tuple + ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) verbose = False @@ -45,8 +51,8 @@ def make_string_literal(b: bytes) -> str: next_code_version = 1 -def get_localsplus(code: types.CodeType): - a = collections.defaultdict(int) +def get_localsplus(code: types.CodeType) -> tuple[tuple[str, ...], bytes]: + a: collections.defaultdict[str, int] = collections.defaultdict(int) for name in code.co_varnames: a[name] |= CO_FAST_LOCAL for name in code.co_cellvars: @@ -58,7 +64,7 @@ def get_localsplus(code: types.CodeType): def get_localsplus_counts(code: types.CodeType, names: Tuple[str, ...], - kinds: bytes) -> Tuple[int, int, int, int]: + kinds: bytes) -> Tuple[int, int, int]: nlocals = 0 ncellvars = 0 nfreevars = 0 @@ -136,7 +142,7 @@ def get_identifiers_and_strings(self) -> tuple[set[str], dict[str, str]]: return identifiers, strings @contextlib.contextmanager - def indent(self) -> None: + def indent(self) -> Iterator[None]: save_level = self.level try: self.level += 1 @@ -148,7 +154,7 @@ def write(self, arg: str) -> None: self.file.writelines((" "*self.level, arg, "\n")) @contextlib.contextmanager - def block(self, prefix: str, suffix: str = "") -> None: + def block(self, prefix: str, suffix: str = "") -> Iterator[None]: self.write(prefix + " {") with self.indent(): yield @@ -250,9 +256,17 @@ def generate_code(self, name: str, code: types.CodeType) -> str: co_names = self.generate(name + "_names", code.co_names) co_filename = self.generate(name + "_filename", code.co_filename) co_name = self.generate(name + "_name", code.co_name) - co_qualname = self.generate(name + "_qualname", code.co_qualname) co_linetable = self.generate(name + "_linetable", code.co_linetable) - co_exceptiontable = self.generate(name + "_exceptiontable", code.co_exceptiontable) + # We use 3.10 for type checking, but this module requires 3.11 + # TODO: bump python version for this script. + co_qualname = self.generate( + name + "_qualname", + code.co_qualname, # type: ignore[attr-defined] + ) + co_exceptiontable = self.generate( + name + "_exceptiontable", + code.co_exceptiontable, # type: ignore[attr-defined] + ) # These fields are not directly accessible localsplusnames, localspluskinds = get_localsplus(code) co_localsplusnames = self.generate(name + "_localsplusnames", localsplusnames) @@ -379,13 +393,13 @@ def generate_complex(self, name: str, z: complex) -> str: self.write(f".cval = {{ {z.real}, {z.imag} }},") return f"&{name}.ob_base" - def generate_frozenset(self, name: str, fs: FrozenSet[object]) -> str: + def generate_frozenset(self, name: str, fs: FrozenSet[Any]) -> str: try: - fs = sorted(fs) + fs_sorted = sorted(fs) except TypeError: # frozen set with incompatible types, fallback to repr() - fs = sorted(fs, key=repr) - ret = self.generate_tuple(name, tuple(fs)) + fs_sorted = sorted(fs, key=repr) + ret = self.generate_tuple(name, tuple(fs_sorted)) self.write("// TODO: The above tuple should be a frozenset") return ret @@ -402,7 +416,7 @@ def generate(self, name: str, obj: object) -> str: # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}") return self.cache[key] self.misses += 1 - if isinstance(obj, (types.CodeType, umarshal.Code)) : + if isinstance(obj, types.CodeType) : val = self.generate_code(name, obj) elif isinstance(obj, tuple): val = self.generate_tuple(name, obj) @@ -458,7 +472,7 @@ def decode_frozen_data(source: str) -> types.CodeType: if re.match(FROZEN_DATA_LINE, line): values.extend([int(x) for x in line.split(",") if x.strip()]) data = bytes(values) - return umarshal.loads(data) + return umarshal.loads(data) # type: ignore[no-any-return] def generate(args: list[str], output: TextIO) -> None: @@ -494,12 +508,12 @@ def generate(args: list[str], output: TextIO) -> None: help="Input file and module name (required) in file:modname format") @contextlib.contextmanager -def report_time(label: str): - t0 = time.time() +def report_time(label: str) -> Iterator[None]: + t0 = time.perf_counter() try: yield finally: - t1 = time.time() + t1 = time.perf_counter() if verbose: print(f"{label}: {t1-t0:.3f} sec") diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index fab35bf68904af..726a37b7695092 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -4,9 +4,11 @@ # .github/workflows/mypy.yml files = Tools/build/compute-changes.py, + Tools/build/deepfreeze.py, Tools/build/generate_sbom.py, Tools/build/verify_ensurepip_wheels.py, - Tools/build/update_file.py + Tools/build/update_file.py, + Tools/build/umarshal.py pretty = True diff --git a/Tools/build/umarshal.py b/Tools/build/umarshal.py index 8198a1780b8dad..e89ba027c5d843 100644 --- a/Tools/build/umarshal.py +++ b/Tools/build/umarshal.py @@ -146,12 +146,12 @@ def r_PyLong(self) -> int: def r_float_bin(self) -> float: buf = self.r_string(8) import struct # Lazy import to avoid breaking UNIX build - return struct.unpack("d", buf)[0] + return struct.unpack("d", buf)[0] # type: ignore[no-any-return] def r_float_str(self) -> float: n = self.r_byte() buf = self.r_string(n) - return ast.literal_eval(buf.decode("ascii")) + return ast.literal_eval(buf.decode("ascii")) # type: ignore[no-any-return] def r_ref_reserve(self, flag: int) -> int: if flag: @@ -307,15 +307,16 @@ def loads(data: bytes) -> Any: return r.r_object() -def main(): +def main() -> None: # Test import marshal, pprint sample = {'foo': {(42, "bar", 3.14)}} data = marshal.dumps(sample) retval = loads(data) assert retval == sample, retval - sample = main.__code__ - data = marshal.dumps(sample) + + sample2 = main.__code__ + data = marshal.dumps(sample2) retval = loads(data) assert isinstance(retval, Code), retval pprint.pprint(retval.__dict__)