diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt index a13f345eece..c4a419c5ffe 100644 --- a/.cspell.dict/python-more.txt +++ b/.cspell.dict/python-more.txt @@ -5,6 +5,7 @@ aexit aiter anext anextawaitable +annotationlib appendleft argcount arrayiterator diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 4667f4ee17b..a03de550681 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -4,7 +4,7 @@ This document provides guidelines for working with GitHub Copilot when contribut ## Project Overview -RustPython is a Python 3 interpreter written in Rust, implementing Python 3.13.0+ compatibility. The project aims to provide: +RustPython is a Python 3 interpreter written in Rust, implementing Python 3.14.0+ compatibility. The project aims to provide: - A complete Python-3 environment entirely in Rust (not CPython bindings) - A clean implementation without compatibility hacks diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b8263026d51..7fc4c653189 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -115,7 +115,7 @@ env: test.test_multiprocessing_spawn.test_processes ENV_POLLUTING_TESTS_WINDOWS: >- # Python version targeted by the CI. - PYTHON_VERSION: "3.13.1" + PYTHON_VERSION: "3.14.2" X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index d48c5e4cfeb..9d549602bc6 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -13,7 +13,7 @@ name: Periodic checks/tasks env: CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,ssl-rustls,jit - PYTHON_VERSION: "3.13.1" + PYTHON_VERSION: "3.14.2" jobs: # codecov collects code coverage data from the rust tests, python snippets and python test suite. diff --git a/.github/workflows/update-doc-db.yml b/.github/workflows/update-doc-db.yml index c580e7d0eaf..37cf56504d2 100644 --- a/.github/workflows/update-doc-db.yml +++ b/.github/workflows/update-doc-db.yml @@ -9,7 +9,7 @@ on: python-version: description: Target python version to generate doc db for type: string - default: "3.13.9" + default: "3.14.2" ref: description: Branch to commit to (leave empty for current branch) type: string diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 82364e9b812..f515bef1a1a 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -25,7 +25,7 @@ RustPython requires the following: stable version: `rustup update stable` - If you do not have Rust installed, use [rustup](https://rustup.rs/) to do so. -- CPython version 3.13 or higher +- CPython version 3.14 or higher - CPython can be installed by your operating system's package manager, from the [Python website](https://www.python.org/downloads/), or using a third-party distribution, such as diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 9eb6f0933b8..d6673f6692f 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,13 +1,16 @@ -from __future__ import annotations -import io import os import sys +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass, field, Field + COLORIZE = True + # types if False: - from typing import IO + from typing import IO, Self, ClassVar + _theme: Theme class ANSIColors: @@ -17,11 +20,13 @@ class ANSIColors: BLUE = "\x1b[34m" CYAN = "\x1b[36m" GREEN = "\x1b[32m" + GREY = "\x1b[90m" MAGENTA = "\x1b[35m" RED = "\x1b[31m" WHITE = "\x1b[37m" # more like LIGHT GRAY YELLOW = "\x1b[33m" + BOLD = "\x1b[1m" BOLD_BLACK = "\x1b[1;30m" # DARK GRAY BOLD_BLUE = "\x1b[1;34m" BOLD_CYAN = "\x1b[1;36m" @@ -60,13 +65,196 @@ class ANSIColors: INTENSE_BACKGROUND_YELLOW = "\x1b[103m" +ColorCodes = set() NoColors = ANSIColors() -for attr in dir(NoColors): +for attr, code in ANSIColors.__dict__.items(): if not attr.startswith("__"): + ColorCodes.add(code) setattr(NoColors, attr, "") +# +# Experimental theming support (see gh-133346) +# + +# - Create a theme by copying an existing `Theme` with one or more sections +# replaced, using `default_theme.copy_with()`; +# - create a theme section by copying an existing `ThemeSection` with one or +# more colors replaced, using for example `default_theme.syntax.copy_with()`; +# - create a theme from scratch by instantiating a `Theme` data class with +# the required sections (which are also dataclass instances). +# +# Then call `_colorize.set_theme(your_theme)` to set it. +# +# Put your theme configuration in $PYTHONSTARTUP for the interactive shell, +# or sitecustomize.py in your virtual environment or Python installation for +# other uses. Your applications can call `_colorize.set_theme()` too. +# +# Note that thanks to the dataclasses providing default values for all fields, +# creating a new theme or theme section from scratch is possible without +# specifying all keys. +# +# For example, here's a theme that makes punctuation and operators less prominent: +# +# try: +# from _colorize import set_theme, default_theme, Syntax, ANSIColors +# except ImportError: +# pass +# else: +# theme_with_dim_operators = default_theme.copy_with( +# syntax=Syntax(op=ANSIColors.INTENSE_BLACK), +# ) +# set_theme(theme_with_dim_operators) +# del set_theme, default_theme, Syntax, ANSIColors, theme_with_dim_operators +# +# Guarding the import ensures that your .pythonstartup file will still work in +# Python 3.13 and older. Deleting the variables ensures they don't remain in your +# interactive shell's global scope. + +class ThemeSection(Mapping[str, str]): + """A mixin/base class for theme sections. + + It enables dictionary access to a section, as well as implements convenience + methods. + """ + + # The two types below are just that: types to inform the type checker that the + # mixin will work in context of those fields existing + __dataclass_fields__: ClassVar[dict[str, Field[str]]] + _name_to_value: Callable[[str], str] + + def __post_init__(self) -> None: + name_to_value = {} + for color_name in self.__dataclass_fields__: + name_to_value[color_name] = getattr(self, color_name) + super().__setattr__('_name_to_value', name_to_value.__getitem__) + + def copy_with(self, **kwargs: str) -> Self: + color_state: dict[str, str] = {} + for color_name in self.__dataclass_fields__: + color_state[color_name] = getattr(self, color_name) + color_state.update(kwargs) + return type(self)(**color_state) + + @classmethod + def no_colors(cls) -> Self: + color_state: dict[str, str] = {} + for color_name in cls.__dataclass_fields__: + color_state[color_name] = "" + return cls(**color_state) + + def __getitem__(self, key: str) -> str: + return self._name_to_value(key) + + def __len__(self) -> int: + return len(self.__dataclass_fields__) + + def __iter__(self) -> Iterator[str]: + return iter(self.__dataclass_fields__) + + +@dataclass(frozen=True, kw_only=True) +class Argparse(ThemeSection): + usage: str = ANSIColors.BOLD_BLUE + prog: str = ANSIColors.BOLD_MAGENTA + prog_extra: str = ANSIColors.MAGENTA + heading: str = ANSIColors.BOLD_BLUE + summary_long_option: str = ANSIColors.CYAN + summary_short_option: str = ANSIColors.GREEN + summary_label: str = ANSIColors.YELLOW + summary_action: str = ANSIColors.GREEN + long_option: str = ANSIColors.BOLD_CYAN + short_option: str = ANSIColors.BOLD_GREEN + label: str = ANSIColors.BOLD_YELLOW + action: str = ANSIColors.BOLD_GREEN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Syntax(ThemeSection): + prompt: str = ANSIColors.BOLD_MAGENTA + keyword: str = ANSIColors.BOLD_BLUE + keyword_constant: str = ANSIColors.BOLD_BLUE + builtin: str = ANSIColors.CYAN + comment: str = ANSIColors.RED + string: str = ANSIColors.GREEN + number: str = ANSIColors.YELLOW + op: str = ANSIColors.RESET + definition: str = ANSIColors.BOLD + soft_keyword: str = ANSIColors.BOLD_BLUE + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Traceback(ThemeSection): + type: str = ANSIColors.BOLD_MAGENTA + message: str = ANSIColors.MAGENTA + filename: str = ANSIColors.MAGENTA + line_no: str = ANSIColors.MAGENTA + frame: str = ANSIColors.MAGENTA + error_highlight: str = ANSIColors.BOLD_RED + error_range: str = ANSIColors.RED + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Unittest(ThemeSection): + passed: str = ANSIColors.GREEN + warn: str = ANSIColors.YELLOW + fail: str = ANSIColors.RED + fail_info: str = ANSIColors.BOLD_RED + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Theme: + """A suite of themes for all sections of Python. + + When adding a new one, remember to also modify `copy_with` and `no_colors` + below. + """ + argparse: Argparse = field(default_factory=Argparse) + syntax: Syntax = field(default_factory=Syntax) + traceback: Traceback = field(default_factory=Traceback) + unittest: Unittest = field(default_factory=Unittest) + + def copy_with( + self, + *, + argparse: Argparse | None = None, + syntax: Syntax | None = None, + traceback: Traceback | None = None, + unittest: Unittest | None = None, + ) -> Self: + """Return a new Theme based on this instance with some sections replaced. + + Themes are immutable to protect against accidental modifications that + could lead to invalid terminal states. + """ + return type(self)( + argparse=argparse or self.argparse, + syntax=syntax or self.syntax, + traceback=traceback or self.traceback, + unittest=unittest or self.unittest, + ) + + @classmethod + def no_colors(cls) -> Self: + """Return a new Theme where colors in all sections are empty strings. + + This allows writing user code as if colors are always used. The color + fields will be ANSI color code strings when colorization is desired + and possible, and empty strings otherwise. + """ + return cls( + argparse=Argparse.no_colors(), + syntax=Syntax.no_colors(), + traceback=Traceback.no_colors(), + unittest=Unittest.no_colors(), + ) + + def get_colors( colorize: bool = False, *, file: IO[str] | IO[bytes] | None = None ) -> ANSIColors: @@ -76,22 +264,37 @@ def get_colors( return NoColors +def decolor(text: str) -> str: + """Remove ANSI color codes from a string.""" + for code in ColorCodes: + text = text.replace(code, "") + return text + + def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: + + def _safe_getenv(k: str, fallback: str | None = None) -> str | None: + """Exception-safe environment retrieval. See gh-128636.""" + try: + return os.environ.get(k, fallback) + except Exception: + return fallback + if file is None: file = sys.stdout if not sys.flags.ignore_environment: - if os.environ.get("PYTHON_COLORS") == "0": + if _safe_getenv("PYTHON_COLORS") == "0": return False - if os.environ.get("PYTHON_COLORS") == "1": + if _safe_getenv("PYTHON_COLORS") == "1": return True - if os.environ.get("NO_COLOR"): + if _safe_getenv("NO_COLOR"): return False if not COLORIZE: return False - if os.environ.get("FORCE_COLOR"): + if _safe_getenv("FORCE_COLOR"): return True - if os.environ.get("TERM") == "dumb": + if _safe_getenv("TERM") == "dumb": return False if not hasattr(file, "fileno"): @@ -108,5 +311,45 @@ def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: try: return os.isatty(file.fileno()) - except io.UnsupportedOperation: + except OSError: return hasattr(file, "isatty") and file.isatty() + + +default_theme = Theme() +theme_no_color = default_theme.no_colors() + + +def get_theme( + *, + tty_file: IO[str] | IO[bytes] | None = None, + force_color: bool = False, + force_no_color: bool = False, +) -> Theme: + """Returns the currently set theme, potentially in a zero-color variant. + + In cases where colorizing is not possible (see `can_colorize`), the returned + theme contains all empty strings in all color definitions. + See `Theme.no_colors()` for more information. + + It is recommended not to cache the result of this function for extended + periods of time because the user might influence theme selection by + the interactive shell, a debugger, or application-specific code. The + environment (including environment variable state and console configuration + on Windows) can also change in the course of the application life cycle. + """ + if force_color or (not force_no_color and + can_colorize(file=tty_file)): + return _theme + return theme_no_color + + +def set_theme(t: Theme) -> None: + global _theme + + if not isinstance(t, Theme): + raise ValueError(f"Expected Theme object, found {t}") + + _theme = t + + +set_theme(default_theme) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py new file mode 100644 index 00000000000..a5788cdbfae --- /dev/null +++ b/Lib/annotationlib.py @@ -0,0 +1,1143 @@ +"""Helpers for introspecting and wrapping annotations.""" + +import ast +import builtins +import enum +import keyword +import sys +import types + +__all__ = [ + "Format", + "ForwardRef", + "call_annotate_function", + "call_evaluate_function", + "get_annotate_from_class_namespace", + "get_annotations", + "annotations_to_string", + "type_repr", +] + + +class Format(enum.IntEnum): + VALUE = 1 + VALUE_WITH_FAKE_GLOBALS = 2 + FORWARDREF = 3 + STRING = 4 + + +_sentinel = object() +# Following `NAME_ERROR_MSG` in `ceval_macros.h`: +_NAME_ERROR_MSG = "name '{name:.200}' is not defined" + + +# Slots shared by ForwardRef and _Stringifier. The __forward__ names must be +# preserved for compatibility with the old typing.ForwardRef class. The remaining +# names are private. +_SLOTS = ( + "__forward_is_argument__", + "__forward_is_class__", + "__forward_module__", + "__weakref__", + "__arg__", + "__globals__", + "__extra_names__", + "__code__", + "__ast_node__", + "__cell__", + "__owner__", + "__stringifier_dict__", +) + + +class ForwardRef: + """Wrapper that holds a forward reference. + + Constructor arguments: + * arg: a string representing the code to be evaluated. + * module: the module where the forward reference was created. + Must be a string, not a module object. + * owner: The owning object (module, class, or function). + * is_argument: Does nothing, retained for compatibility. + * is_class: True if the forward reference was created in class scope. + + """ + + __slots__ = _SLOTS + + def __init__( + self, + arg, + *, + module=None, + owner=None, + is_argument=True, + is_class=False, + ): + if not isinstance(arg, str): + raise TypeError(f"Forward reference must be a string -- got {arg!r}") + + self.__arg__ = arg + self.__forward_is_argument__ = is_argument + self.__forward_is_class__ = is_class + self.__forward_module__ = module + self.__owner__ = owner + # These are always set to None here but may be non-None if a ForwardRef + # is created through __class__ assignment on a _Stringifier object. + self.__globals__ = None + # This may be either a cell object (for a ForwardRef referring to a single name) + # or a dict mapping cell names to cell objects (for a ForwardRef containing references + # to multiple names). + self.__cell__ = None + self.__extra_names__ = None + # These are initially None but serve as a cache and may be set to a non-None + # value later. + self.__code__ = None + self.__ast_node__ = None + + def __init_subclass__(cls, /, *args, **kwds): + raise TypeError("Cannot subclass ForwardRef") + + def evaluate( + self, + *, + globals=None, + locals=None, + type_params=None, + owner=None, + format=Format.VALUE, + ): + """Evaluate the forward reference and return the value. + + If the forward reference cannot be evaluated, raise an exception. + """ + match format: + case Format.STRING: + return self.__forward_arg__ + case Format.VALUE: + is_forwardref_format = False + case Format.FORWARDREF: + is_forwardref_format = True + case _: + raise NotImplementedError(format) + if isinstance(self.__cell__, types.CellType): + try: + return self.__cell__.cell_contents + except ValueError: + pass + if owner is None: + owner = self.__owner__ + + if globals is None and self.__forward_module__ is not None: + globals = getattr( + sys.modules.get(self.__forward_module__, None), "__dict__", None + ) + if globals is None: + globals = self.__globals__ + if globals is None: + if isinstance(owner, type): + module_name = getattr(owner, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + globals = getattr(module, "__dict__", None) + elif isinstance(owner, types.ModuleType): + globals = getattr(owner, "__dict__", None) + elif callable(owner): + globals = getattr(owner, "__globals__", None) + + # If we pass None to eval() below, the globals of this module are used. + if globals is None: + globals = {} + + if type_params is None and owner is not None: + type_params = getattr(owner, "__type_params__", None) + + if locals is None: + locals = {} + if isinstance(owner, type): + locals.update(vars(owner)) + elif ( + type_params is not None + or isinstance(self.__cell__, dict) + or self.__extra_names__ + ): + # Create a new locals dict if necessary, + # to avoid mutating the argument. + locals = dict(locals) + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params is not None: + for param in type_params: + locals.setdefault(param.__name__, param) + + # Similar logic can be used for nonlocals, which should not + # override locals. + if isinstance(self.__cell__, dict): + for cell_name, cell in self.__cell__.items(): + try: + cell_value = cell.cell_contents + except ValueError: + pass + else: + locals.setdefault(cell_name, cell_value) + + if self.__extra_names__: + locals.update(self.__extra_names__) + + arg = self.__forward_arg__ + if arg.isidentifier() and not keyword.iskeyword(arg): + if arg in locals: + return locals[arg] + elif arg in globals: + return globals[arg] + elif hasattr(builtins, arg): + return getattr(builtins, arg) + elif is_forwardref_format: + return self + else: + raise NameError(_NAME_ERROR_MSG.format(name=arg), name=arg) + else: + code = self.__forward_code__ + try: + return eval(code, globals=globals, locals=locals) + except Exception: + if not is_forwardref_format: + raise + + # All variables, in scoping order, should be checked before + # triggering __missing__ to create a _Stringifier. + new_locals = _StringifierDict( + {**builtins.__dict__, **globals, **locals}, + globals=globals, + owner=owner, + is_class=self.__forward_is_class__, + format=format, + ) + try: + result = eval(code, globals=globals, locals=new_locals) + except Exception: + return self + else: + new_locals.transmogrify(self.__cell__) + return result + + def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): + import typing + import warnings + + if type_params is _sentinel: + typing._deprecation_warning_for_no_type_params_passed( + "typing.ForwardRef._evaluate" + ) + type_params = () + warnings._deprecated( + "ForwardRef._evaluate", + "{name} is a private API and is retained for compatibility, but will be removed" + " in Python 3.16. Use ForwardRef.evaluate() or typing.evaluate_forward_ref() instead.", + remove=(3, 16), + ) + return typing.evaluate_forward_ref( + self, + globals=globalns, + locals=localns, + type_params=type_params, + _recursive_guard=recursive_guard, + ) + + @property + def __forward_arg__(self): + if self.__arg__ is not None: + return self.__arg__ + if self.__ast_node__ is not None: + self.__arg__ = ast.unparse(self.__ast_node__) + return self.__arg__ + raise AssertionError( + "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" + ) + + @property + def __forward_code__(self): + if self.__code__ is not None: + return self.__code__ + arg = self.__forward_arg__ + try: + self.__code__ = compile(_rewrite_star_unpack(arg), "", "eval") + except SyntaxError: + raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") + return self.__code__ + + def __eq__(self, other): + if not isinstance(other, ForwardRef): + return NotImplemented + return ( + self.__forward_arg__ == other.__forward_arg__ + and self.__forward_module__ == other.__forward_module__ + # Use "is" here because we use id() for this in __hash__ + # because dictionaries are not hashable. + and self.__globals__ is other.__globals__ + and self.__forward_is_class__ == other.__forward_is_class__ + and self.__cell__ == other.__cell__ + and self.__owner__ == other.__owner__ + and ( + (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == + (tuple(sorted(other.__extra_names__.items())) if other.__extra_names__ else None) + ) + ) + + def __hash__(self): + return hash(( + self.__forward_arg__, + self.__forward_module__, + id(self.__globals__), # dictionaries are not hashable, so hash by identity + self.__forward_is_class__, + tuple(sorted(self.__cell__.items())) if isinstance(self.__cell__, dict) else self.__cell__, + self.__owner__, + tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, + )) + + def __or__(self, other): + return types.UnionType[self, other] + + def __ror__(self, other): + return types.UnionType[other, self] + + def __repr__(self): + extra = [] + if self.__forward_module__ is not None: + extra.append(f", module={self.__forward_module__!r}") + if self.__forward_is_class__: + extra.append(", is_class=True") + if self.__owner__ is not None: + extra.append(f", owner={self.__owner__!r}") + return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})" + + +_Template = type(t"") + + +class _Stringifier: + # Must match the slots on ForwardRef, so we can turn an instance of one into an + # instance of the other in place. + __slots__ = _SLOTS + + def __init__( + self, + node, + globals=None, + owner=None, + is_class=False, + cell=None, + *, + stringifier_dict, + extra_names=None, + ): + # Either an AST node or a simple str (for the common case where a ForwardRef + # represent a single name). + assert isinstance(node, (ast.AST, str)) + self.__arg__ = None + self.__forward_is_argument__ = False + self.__forward_is_class__ = is_class + self.__forward_module__ = None + self.__code__ = None + self.__ast_node__ = node + self.__globals__ = globals + self.__extra_names__ = extra_names + self.__cell__ = cell + self.__owner__ = owner + self.__stringifier_dict__ = stringifier_dict + + def __convert_to_ast(self, other): + if isinstance(other, _Stringifier): + if isinstance(other.__ast_node__, str): + return ast.Name(id=other.__ast_node__), other.__extra_names__ + return other.__ast_node__, other.__extra_names__ + elif type(other) is _Template: + return _template_to_ast(other), None + elif ( + # In STRING format we don't bother with the create_unique_name() dance; + # it's better to emit the repr() of the object instead of an opaque name. + self.__stringifier_dict__.format == Format.STRING + or other is None + or type(other) in (str, int, float, bool, complex) + ): + return ast.Constant(value=other), None + elif type(other) is dict: + extra_names = {} + keys = [] + values = [] + for key, value in other.items(): + new_key, new_extra_names = self.__convert_to_ast(key) + if new_extra_names is not None: + extra_names.update(new_extra_names) + keys.append(new_key) + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + values.append(new_value) + return ast.Dict(keys, values), extra_names + elif type(other) in (list, tuple, set): + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + ast_class = {list: ast.List, tuple: ast.Tuple, set: ast.Set}[type(other)] + return ast_class(elts), extra_names + else: + name = self.__stringifier_dict__.create_unique_name() + return ast.Name(id=name), {name: other} + + def __convert_to_ast_getitem(self, other): + if isinstance(other, slice): + extra_names = {} + + def conv(obj): + if obj is None: + return None + new_obj, new_extra_names = self.__convert_to_ast(obj) + if new_extra_names is not None: + extra_names.update(new_extra_names) + return new_obj + + return ast.Slice( + lower=conv(other.start), + upper=conv(other.stop), + step=conv(other.step), + ), extra_names + else: + return self.__convert_to_ast(other) + + def __get_ast(self): + node = self.__ast_node__ + if isinstance(node, str): + return ast.Name(id=node) + return node + + def __make_new(self, node, extra_names=None): + new_extra_names = {} + if self.__extra_names__ is not None: + new_extra_names.update(self.__extra_names__) + if extra_names is not None: + new_extra_names.update(extra_names) + stringifier = _Stringifier( + node, + self.__globals__, + self.__owner__, + self.__forward_is_class__, + stringifier_dict=self.__stringifier_dict__, + extra_names=new_extra_names or None, + ) + self.__stringifier_dict__.stringifiers.append(stringifier) + return stringifier + + # Must implement this since we set __eq__. We hash by identity so that + # stringifiers in dict keys are kept separate. + def __hash__(self): + return id(self) + + def __getitem__(self, other): + # Special case, to avoid stringifying references to class-scoped variables + # as '__classdict__["x"]'. + if self.__ast_node__ == "__classdict__": + raise KeyError + if isinstance(other, tuple): + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast_getitem(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + other = ast.Tuple(elts) + else: + other, extra_names = self.__convert_to_ast_getitem(other) + assert isinstance(other, ast.AST), repr(other) + return self.__make_new(ast.Subscript(self.__get_ast(), other), extra_names) + + def __getattr__(self, attr): + return self.__make_new(ast.Attribute(self.__get_ast(), attr)) + + def __call__(self, *args, **kwargs): + extra_names = {} + ast_args = [] + for arg in args: + new_arg, new_extra_names = self.__convert_to_ast(arg) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_args.append(new_arg) + ast_kwargs = [] + for key, value in kwargs.items(): + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_kwargs.append(ast.keyword(key, new_value)) + return self.__make_new(ast.Call(self.__get_ast(), ast_args, ast_kwargs), extra_names) + + def __iter__(self): + yield self.__make_new(ast.Starred(self.__get_ast())) + + def __repr__(self): + if isinstance(self.__ast_node__, str): + return self.__ast_node__ + return ast.unparse(self.__ast_node__) + + def __format__(self, format_spec): + raise TypeError("Cannot stringify annotation containing string formatting") + + def _make_binop(op: ast.AST): + def binop(self, other): + rhs, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.BinOp(self.__get_ast(), op, rhs), extra_names + ) + + return binop + + __add__ = _make_binop(ast.Add()) + __sub__ = _make_binop(ast.Sub()) + __mul__ = _make_binop(ast.Mult()) + __matmul__ = _make_binop(ast.MatMult()) + __truediv__ = _make_binop(ast.Div()) + __mod__ = _make_binop(ast.Mod()) + __lshift__ = _make_binop(ast.LShift()) + __rshift__ = _make_binop(ast.RShift()) + __or__ = _make_binop(ast.BitOr()) + __xor__ = _make_binop(ast.BitXor()) + __and__ = _make_binop(ast.BitAnd()) + __floordiv__ = _make_binop(ast.FloorDiv()) + __pow__ = _make_binop(ast.Pow()) + + del _make_binop + + def _make_rbinop(op: ast.AST): + def rbinop(self, other): + new_other, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.BinOp(new_other, op, self.__get_ast()), extra_names + ) + + return rbinop + + __radd__ = _make_rbinop(ast.Add()) + __rsub__ = _make_rbinop(ast.Sub()) + __rmul__ = _make_rbinop(ast.Mult()) + __rmatmul__ = _make_rbinop(ast.MatMult()) + __rtruediv__ = _make_rbinop(ast.Div()) + __rmod__ = _make_rbinop(ast.Mod()) + __rlshift__ = _make_rbinop(ast.LShift()) + __rrshift__ = _make_rbinop(ast.RShift()) + __ror__ = _make_rbinop(ast.BitOr()) + __rxor__ = _make_rbinop(ast.BitXor()) + __rand__ = _make_rbinop(ast.BitAnd()) + __rfloordiv__ = _make_rbinop(ast.FloorDiv()) + __rpow__ = _make_rbinop(ast.Pow()) + + del _make_rbinop + + def _make_compare(op): + def compare(self, other): + rhs, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.Compare( + left=self.__get_ast(), + ops=[op], + comparators=[rhs], + ), + extra_names, + ) + + return compare + + __lt__ = _make_compare(ast.Lt()) + __le__ = _make_compare(ast.LtE()) + __eq__ = _make_compare(ast.Eq()) + __ne__ = _make_compare(ast.NotEq()) + __gt__ = _make_compare(ast.Gt()) + __ge__ = _make_compare(ast.GtE()) + + del _make_compare + + def _make_unary_op(op): + def unary_op(self): + return self.__make_new(ast.UnaryOp(op, self.__get_ast())) + + return unary_op + + __invert__ = _make_unary_op(ast.Invert()) + __pos__ = _make_unary_op(ast.UAdd()) + __neg__ = _make_unary_op(ast.USub()) + + del _make_unary_op + + +def _template_to_ast_constructor(template): + """Convert a `template` instance to a non-literal AST.""" + args = [] + for part in template: + match part: + case str(): + args.append(ast.Constant(value=part)) + case _: + interp = ast.Call( + func=ast.Name(id="Interpolation"), + args=[ + ast.Constant(value=part.value), + ast.Constant(value=part.expression), + ast.Constant(value=part.conversion), + ast.Constant(value=part.format_spec), + ] + ) + args.append(interp) + return ast.Call(func=ast.Name(id="Template"), args=args, keywords=[]) + + +def _template_to_ast_literal(template, parsed): + """Convert a `template` instance to a t-string literal AST.""" + values = [] + interp_count = 0 + for part in template: + match part: + case str(): + values.append(ast.Constant(value=part)) + case _: + interp = ast.Interpolation( + str=part.expression, + value=parsed[interp_count], + conversion=ord(part.conversion) if part.conversion else -1, + format_spec=ast.Constant(value=part.format_spec) + if part.format_spec + else None, + ) + values.append(interp) + interp_count += 1 + return ast.TemplateStr(values=values) + + +def _template_to_ast(template): + """Make a best-effort conversion of a `template` instance to an AST.""" + # gh-138558: Not all Template instances can be represented as t-string + # literals. Return the most accurate AST we can. See issue for details. + + # If any expr is empty or whitespace only, we cannot convert to a literal. + if any(part.expression.strip() == "" for part in template.interpolations): + return _template_to_ast_constructor(template) + + try: + # Wrap in parens to allow whitespace inside interpolation curly braces + parsed = tuple( + ast.parse(f"({part.expression})", mode="eval").body + for part in template.interpolations + ) + except SyntaxError: + return _template_to_ast_constructor(template) + + return _template_to_ast_literal(template, parsed) + + +class _StringifierDict(dict): + def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format): + super().__init__(namespace) + self.namespace = namespace + self.globals = globals + self.owner = owner + self.is_class = is_class + self.stringifiers = [] + self.next_id = 1 + self.format = format + + def __missing__(self, key): + fwdref = _Stringifier( + key, + globals=self.globals, + owner=self.owner, + is_class=self.is_class, + stringifier_dict=self, + ) + self.stringifiers.append(fwdref) + return fwdref + + def transmogrify(self, cell_dict): + for obj in self.stringifiers: + obj.__class__ = ForwardRef + obj.__stringifier_dict__ = None # not needed for ForwardRef + if isinstance(obj.__ast_node__, str): + obj.__arg__ = obj.__ast_node__ + obj.__ast_node__ = None + if cell_dict is not None and obj.__cell__ is None: + obj.__cell__ = cell_dict + + def create_unique_name(self): + name = f"__annotationlib_name_{self.next_id}__" + self.next_id += 1 + return name + + +def call_evaluate_function(evaluate, format, *, owner=None): + """Call an evaluate function. Evaluate functions are normally generated for + the value of type aliases and the bounds, constraints, and defaults of + type parameter objects. + """ + return call_annotate_function(evaluate, format, owner=owner, _is_evaluate=True) + + +def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): + """Call an __annotate__ function. __annotate__ functions are normally + generated by the compiler to defer the evaluation of annotations. They + can be called with any of the format arguments in the Format enum, but + compiler-generated __annotate__ functions only support the VALUE format. + This function provides additional functionality to call __annotate__ + functions with the FORWARDREF and STRING formats. + + *annotate* must be an __annotate__ function, which takes a single argument + and returns a dict of annotations. + + *format* must be a member of the Format enum or one of the corresponding + integer values. + + *owner* can be the object that owns the annotations (i.e., the module, + class, or function that the __annotate__ function derives from). With the + FORWARDREF format, it is used to provide better evaluation capabilities + on the generated ForwardRef objects. + + """ + if format == Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") + try: + return annotate(format) + except NotImplementedError: + pass + if format == Format.STRING: + # STRING is implemented by calling the annotate function in a special + # environment where every name lookup results in an instance of _Stringifier. + # _Stringifier supports every dunder operation and returns a new _Stringifier. + # At the end, we get a dictionary that mostly contains _Stringifier objects (or + # possibly constants if the annotate function uses them directly). We then + # convert each of those into a string to get an approximation of the + # original source. + + # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented + # See: https://github.com/python/cpython/issues/138764 + # Only fail on NotImplementedError + try: + annotate(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented: fallback to VALUE + return annotations_to_string(annotate(Format.VALUE)) + except Exception: + pass + + globals = _StringifierDict({}, format=format) + is_class = isinstance(owner, type) + closure, _ = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=False + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + annos = func(Format.VALUE_WITH_FAKE_GLOBALS) + if _is_evaluate: + return _stringify_single(annos) + return { + key: _stringify_single(val) + for key, val in annos.items() + } + elif format == Format.FORWARDREF: + # FORWARDREF is implemented similarly to STRING, but there are two changes, + # at the beginning and the end of the process. + # First, while STRING uses an empty dictionary as the namespace, so that all + # name lookups result in _Stringifier objects, FORWARDREF uses the globals + # and builtins, so that defined names map to their real values. + # Second, instead of returning strings, we want to return either real values + # or ForwardRef objects. To do this, we keep track of all _Stringifier objects + # created while the annotation is being evaluated, and at the end we convert + # them all to ForwardRef objects by assigning to __class__. To make this + # technique work, we have to ensure that the _Stringifier and ForwardRef + # classes share the same attributes. + # We use this technique because while the annotations are being evaluated, + # we want to support all operations that the language allows, including even + # __getattr__ and __eq__, and return new _Stringifier objects so we can accurately + # reconstruct the source. But in the dictionary that we eventually return, we + # want to return objects with more user-friendly behavior, such as an __eq__ + # that returns a bool and an defined set of attributes. + namespace = {**annotate.__builtins__, **annotate.__globals__} + is_class = isinstance(owner, type) + globals = _StringifierDict( + namespace, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, + ) + closure, cell_dict = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=True + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + try: + result = func(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE + return annotate(Format.VALUE) + except Exception: + pass + else: + globals.transmogrify(cell_dict) + return result + + # Try again, but do not provide any globals. This allows us to return + # a value in certain cases where an exception gets raised during evaluation. + globals = _StringifierDict( + {}, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, + ) + closure, cell_dict = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=False + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + result = func(Format.VALUE_WITH_FAKE_GLOBALS) + globals.transmogrify(cell_dict) + if _is_evaluate: + if isinstance(result, ForwardRef): + return result.evaluate(format=Format.FORWARDREF) + else: + return result + else: + return { + key: ( + val.evaluate(format=Format.FORWARDREF) + if isinstance(val, ForwardRef) + else val + ) + for key, val in result.items() + } + elif format == Format.VALUE: + # Should be impossible because __annotate__ functions must not raise + # NotImplementedError for this format. + raise RuntimeError("annotate function does not support VALUE format") + else: + raise ValueError(f"Invalid format: {format!r}") + + +def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): + if not annotate.__closure__: + return None, None + new_closure = [] + cell_dict = {} + for name, cell in zip(annotate.__code__.co_freevars, annotate.__closure__, strict=True): + cell_dict[name] = cell + new_cell = None + if allow_evaluation: + try: + cell.cell_contents + except ValueError: + pass + else: + new_cell = cell + if new_cell is None: + fwdref = _Stringifier( + name, + cell=cell, + owner=owner, + globals=annotate.__globals__, + is_class=is_class, + stringifier_dict=stringifier_dict, + ) + stringifier_dict.stringifiers.append(fwdref) + new_cell = types.CellType(fwdref) + new_closure.append(new_cell) + return tuple(new_closure), cell_dict + + +def _stringify_single(anno): + if anno is ...: + return "..." + # We have to handle str specially to support PEP 563 stringified annotations. + elif isinstance(anno, str): + return anno + elif isinstance(anno, _Template): + return ast.unparse(_template_to_ast(anno)) + else: + return repr(anno) + + +def get_annotate_from_class_namespace(obj): + """Retrieve the annotate function from a class namespace dictionary. + + Return None if the namespace does not contain an annotate function. + This is useful in metaclass ``__new__`` methods to retrieve the annotate function. + """ + try: + return obj["__annotate__"] + except KeyError: + return obj.get("__annotate_func__", None) + + +def get_annotations( + obj, *, globals=None, locals=None, eval_str=False, format=Format.VALUE +): + """Compute the annotations dict for an object. + + obj may be a callable, class, module, or other object with + __annotate__ or __annotations__ attributes. + Passing any other object raises TypeError. + + The *format* parameter controls the format in which annotations are returned, + and must be a member of the Format enum or its integer equivalent. + For the VALUE format, the __annotations__ is tried first; if it + does not exist, the __annotate__ function is called. The + FORWARDREF format uses __annotations__ if it exists and can be + evaluated, and otherwise falls back to calling the __annotate__ function. + The SOURCE format tries __annotate__ first, and falls back to + using __annotations__, stringified using annotations_to_string(). + + This function handles several details for you: + + * If eval_str is true, values of type str will + be un-stringized using eval(). This is intended + for use with stringized annotations + ("from __future__ import annotations"). + * If obj doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using getattr() and dict.get() for safety. + * Always, always, always returns a freshly-created dict. + + eval_str controls whether or not values of type str are replaced + with the result of calling eval() on those values: + + * If eval_str is true, eval() is called on values of type str. + * If eval_str is false (the default), values of type str are unchanged. + + globals and locals are passed in to eval(); see the documentation + for eval() for more information. If either globals or locals is + None, this function may replace that value with a context-specific + default, contingent on type(obj): + + * If obj is a module, globals defaults to obj.__dict__. + * If obj is a class, globals defaults to + sys.modules[obj.__module__].__dict__ and locals + defaults to the obj class namespace. + * If obj is a callable, globals defaults to obj.__globals__, + although if obj is a wrapped function (using + functools.update_wrapper()) it is first unwrapped. + """ + if eval_str and format != Format.VALUE: + raise ValueError("eval_str=True is only supported with format=Format.VALUE") + + match format: + case Format.VALUE: + # For VALUE, we first look at __annotations__ + ann = _get_dunder_annotations(obj) + + # If it's not there, try __annotate__ instead + if ann is None: + ann = _get_and_call_annotate(obj, format) + case Format.FORWARDREF: + # For FORWARDREF, we use __annotations__ if it exists + try: + ann = _get_dunder_annotations(obj) + except Exception: + pass + else: + if ann is not None: + return dict(ann) + + # But if __annotations__ threw a NameError, we try calling __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is None: + # If that didn't work either, we have a very weird object: evaluating + # __annotations__ threw NameError and there is no __annotate__. In that case, + # we fall back to trying __annotations__ again. + ann = _get_dunder_annotations(obj) + case Format.STRING: + # For STRING, we try to call __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is not None: + return dict(ann) + # But if we didn't get it, we use __annotations__ instead. + ann = _get_dunder_annotations(obj) + if ann is not None: + return annotations_to_string(ann) + case Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") + case _: + raise ValueError(f"Unsupported format {format!r}") + + if ann is None: + if isinstance(obj, type) or callable(obj): + return {} + raise TypeError(f"{obj!r} does not have annotations") + + if not ann: + return {} + + if not eval_str: + return dict(ann) + + if globals is None or locals is None: + if isinstance(obj, type): + # class + obj_globals = None + module_name = getattr(obj, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + obj_globals = getattr(module, "__dict__", None) + obj_locals = dict(vars(obj)) + unwrap = obj + elif isinstance(obj, types.ModuleType): + # module + obj_globals = getattr(obj, "__dict__") + obj_locals = None + unwrap = None + elif callable(obj): + # this includes types.Function, types.BuiltinFunctionType, + # types.BuiltinMethodType, functools.partial, functools.singledispatch, + # "class funclike" from Lib/test/test_inspect... on and on it goes. + obj_globals = getattr(obj, "__globals__", None) + obj_locals = None + unwrap = obj + else: + obj_globals = obj_locals = unwrap = None + + if unwrap is not None: + while True: + if hasattr(unwrap, "__wrapped__"): + unwrap = unwrap.__wrapped__ + continue + if functools := sys.modules.get("functools"): + if isinstance(unwrap, functools.partial): + unwrap = unwrap.func + continue + break + if hasattr(unwrap, "__globals__"): + obj_globals = unwrap.__globals__ + + if globals is None: + globals = obj_globals + if locals is None: + locals = obj_locals + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params := getattr(obj, "__type_params__", ()): + if locals is None: + locals = {} + locals = {param.__name__: param for param in type_params} | locals + + return_value = { + key: value if not isinstance(value, str) + else eval(_rewrite_star_unpack(value), globals, locals) + for key, value in ann.items() + } + return return_value + + +def type_repr(value): + """Convert a Python value to a format suitable for use with the STRING format. + + This is intended as a helper for tools that support the STRING format but do + not have access to the code that originally produced the annotations. It uses + repr() for most objects. + + """ + if isinstance(value, (type, types.FunctionType, types.BuiltinFunctionType)): + if value.__module__ == "builtins": + return value.__qualname__ + return f"{value.__module__}.{value.__qualname__}" + elif isinstance(value, _Template): + tree = _template_to_ast(value) + return ast.unparse(tree) + if value is ...: + return "..." + return repr(value) + + +def annotations_to_string(annotations): + """Convert an annotation dict containing values to approximately the STRING format. + + Always returns a fresh a dictionary. + """ + return { + n: t if isinstance(t, str) else type_repr(t) + for n, t in annotations.items() + } + + +def _rewrite_star_unpack(arg): + """If the given argument annotation expression is a star unpack e.g. `'*Ts'` + rewrite it to a valid expression. + """ + if arg.startswith("*"): + return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] + else: + return arg + + +def _get_and_call_annotate(obj, format): + """Get the __annotate__ function and call it. + + May not return a fresh dictionary. + """ + annotate = getattr(obj, "__annotate__", None) + if annotate is not None: + ann = call_annotate_function(annotate, format, owner=obj) + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotate__ returned a non-dict") + return ann + return None + + +_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__ + + +def _get_dunder_annotations(obj): + """Return the annotations for an object, checking that it is a dictionary. + + Does not return a fresh dictionary. + """ + # This special case is needed to support types defined under + # from __future__ import annotations, where accessing the __annotations__ + # attribute directly might return annotations for the wrong class. + if isinstance(obj, type): + try: + ann = _BASE_GET_ANNOTATIONS(obj) + except AttributeError: + # For static types, the descriptor raises AttributeError. + return None + else: + ann = getattr(obj, "__annotations__", None) + if ann is None: + return None + + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + return ann diff --git a/Lib/argparse.py b/Lib/argparse.py index bd088ea0e66..88c1f5a7ef3 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -18,11 +18,12 @@ 'integers', metavar='int', nargs='+', type=int, help='an integer to be summed') parser.add_argument( - '--log', default=sys.stdout, type=argparse.FileType('w'), + '--log', help='the file where the sum should be written') args = parser.parse_args() - args.log.write('%s' % sum(args.integers)) - args.log.close() + with (open(args.log, 'w') if args.log is not None + else contextlib.nullcontext(sys.stdout)) as log: + log.write('%s' % sum(args.integers)) The module contains the following public classes: @@ -39,7 +40,8 @@ - FileType -- A factory for defining types of files to be created. As the example above shows, instances of FileType are typically passed as - the type= argument of add_argument() calls. + the type= argument of add_argument() calls. Deprecated since + Python 3.14. - Action -- The base class for parser actions. Typically actions are selected by passing strings like 'store_true' or 'append_const' to @@ -159,18 +161,21 @@ class HelpFormatter(object): provided by the class are considered an implementation detail. """ - def __init__(self, - prog, - indent_increment=2, - max_help_position=24, - width=None): - + def __init__( + self, + prog, + indent_increment=2, + max_help_position=24, + width=None, + color=True, + ): # default setting for width if width is None: import shutil width = shutil.get_terminal_size().columns width -= 2 + self._set_color(color) self._prog = prog self._indent_increment = indent_increment self._max_help_position = min(max_help_position, @@ -187,6 +192,16 @@ def __init__(self, self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII) self._long_break_matcher = _re.compile(r'\n\n\n+') + def _set_color(self, color): + from _colorize import can_colorize, decolor, get_theme + + if color and can_colorize(): + self._theme = get_theme(force_color=True).argparse + self._decolor = decolor + else: + self._theme = get_theme(force_no_color=True).argparse + self._decolor = lambda text: text + # =============================== # Section and indentation methods # =============================== @@ -225,7 +240,11 @@ def format_help(self): if self.heading is not SUPPRESS and self.heading is not None: current_indent = self.formatter._current_indent heading_text = _('%(heading)s:') % dict(heading=self.heading) - heading = '%*s%s\n' % (current_indent, '', heading_text) + t = self.formatter._theme + heading = ( + f'{" " * current_indent}' + f'{t.heading}{heading_text}{t.reset}\n' + ) else: heading = '' @@ -262,7 +281,7 @@ def add_argument(self, action): if action.help is not SUPPRESS: # find all invocations - get_invocation = self._format_action_invocation + get_invocation = lambda x: self._decolor(self._format_action_invocation(x)) invocation_lengths = [len(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent) @@ -296,16 +315,23 @@ def _join_parts(self, part_strings): if part and part is not SUPPRESS]) def _format_usage(self, usage, actions, groups, prefix): + t = self._theme + if prefix is None: prefix = _('usage: ') # if usage is specified, use that if usage is not None: - usage = usage % dict(prog=self._prog) + usage = ( + t.prog_extra + + usage + % {"prog": f"{t.prog}{self._prog}{t.reset}{t.prog_extra}"} + + t.reset + ) # if no optionals or positionals are available, usage is just prog elif usage is None and not actions: - usage = '%(prog)s' % dict(prog=self._prog) + usage = f"{t.prog}{self._prog}{t.reset}" # if optionals and positionals are available, calculate usage elif usage is None: @@ -327,7 +353,7 @@ def _format_usage(self, usage, actions, groups, prefix): # wrap the usage parts if it's too long text_width = self._width - self._current_indent - if len(prefix) + len(usage) > text_width: + if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts opt_parts = self._get_actions_usage_parts(optionals, groups) @@ -343,12 +369,13 @@ def get_lines(parts, indent, prefix=None): else: line_len = indent_length - 1 for part in parts: - if line_len + 1 + len(part) > text_width and line: + part_len = len(self._decolor(part)) + if line_len + 1 + part_len > text_width and line: lines.append(indent + ' '.join(line)) line = [] line_len = indent_length - 1 line.append(part) - line_len += len(part) + 1 + line_len += part_len + 1 if line: lines.append(indent + ' '.join(line)) if prefix is not None: @@ -356,8 +383,9 @@ def get_lines(parts, indent, prefix=None): return lines # if prog is short, follow it with optionals or positionals - if len(prefix) + len(prog) <= 0.75 * text_width: - indent = ' ' * (len(prefix) + len(prog) + 1) + prog_len = len(self._decolor(prog)) + if len(prefix) + prog_len <= 0.75 * text_width: + indent = ' ' * (len(prefix) + prog_len + 1) if opt_parts: lines = get_lines([prog] + opt_parts, indent, prefix) lines.extend(get_lines(pos_parts, indent)) @@ -380,12 +408,18 @@ def get_lines(parts, indent, prefix=None): # join lines into usage usage = '\n'.join(lines) + usage = usage.removeprefix(prog) + usage = f"{t.prog}{prog}{t.reset}{usage}" + # prefix with 'usage:' - return '%s%s\n\n' % (prefix, usage) + return f'{t.usage}{prefix}{t.reset}{usage}\n\n' def _format_actions_usage(self, actions, groups): return ' '.join(self._get_actions_usage_parts(actions, groups)) + def _is_long_option(self, string): + return len(string) > 2 + def _get_actions_usage_parts(self, actions, groups): # find group indices and identify actions in groups group_actions = set() @@ -398,17 +432,18 @@ def _get_actions_usage_parts(self, actions, groups): continue try: - start = actions.index(group._group_actions[0]) + start = min(actions.index(item) for item in group._group_actions) except ValueError: continue else: end = start + len(group._group_actions) - if actions[start:end] == group._group_actions: + if set(actions[start:end]) == set(group._group_actions): group_actions.update(group._group_actions) inserts[start, end] = group # collect all actions format strings parts = [] + t = self._theme for action in actions: # suppressed arguments are marked with None @@ -418,7 +453,11 @@ def _get_actions_usage_parts(self, actions, groups): # produce all arg strings elif not action.option_strings: default = self._get_default_metavar_for_positional(action) - part = self._format_args(action, default) + part = ( + t.summary_action + + self._format_args(action, default) + + t.reset + ) # if it's in a group, strip the outer [] if action in group_actions: @@ -428,18 +467,26 @@ def _get_actions_usage_parts(self, actions, groups): # produce the first way to invoke the option in brackets else: option_string = action.option_strings[0] + if self._is_long_option(option_string): + option_color = t.summary_long_option + else: + option_color = t.summary_short_option # if the Optional doesn't take a value, format is: # -s or --long if action.nargs == 0: part = action.format_usage() + part = f"{option_color}{part}{t.reset}" # if the Optional takes a value, format is: # -s ARGS or --long ARGS else: default = self._get_default_metavar_for_optional(action) args_string = self._format_args(action, default) - part = '%s %s' % (option_string, args_string) + part = ( + f"{option_color}{option_string} " + f"{t.summary_label}{args_string}{t.reset}" + ) # make it look optional if it's not required or in a group if not action.required and action not in group_actions: @@ -486,6 +533,7 @@ def _format_action(self, action): help_width = max(self._width - help_position, 11) action_width = help_position - self._current_indent - 2 action_header = self._format_action_invocation(action) + action_header_no_color = self._decolor(action_header) # no help; start on same line and add a final newline if not action.help: @@ -493,9 +541,15 @@ def _format_action(self, action): action_header = '%*s%s\n' % tup # short action name; start on the same line and pad two spaces - elif len(action_header) <= action_width: - tup = self._current_indent, '', action_width, action_header + elif len(action_header_no_color) <= action_width: + # calculate widths without color codes + action_header_color = action_header + tup = self._current_indent, '', action_width, action_header_no_color action_header = '%*s%-*s ' % tup + # swap in the colored header + action_header = action_header.replace( + action_header_no_color, action_header_color + ) indent_first = 0 # long action name; start on the next line @@ -528,23 +582,42 @@ def _format_action(self, action): return self._join_parts(parts) def _format_action_invocation(self, action): + t = self._theme + if not action.option_strings: default = self._get_default_metavar_for_positional(action) - return ' '.join(self._metavar_formatter(action, default)(1)) + return ( + t.action + + ' '.join(self._metavar_formatter(action, default)(1)) + + t.reset + ) else: + def color_option_strings(strings): + parts = [] + for s in strings: + if self._is_long_option(s): + parts.append(f"{t.long_option}{s}{t.reset}") + else: + parts.append(f"{t.short_option}{s}{t.reset}") + return parts + # if the Optional doesn't take a value, format is: # -s, --long if action.nargs == 0: - return ', '.join(action.option_strings) + option_strings = color_option_strings(action.option_strings) + return ', '.join(option_strings) # if the Optional takes a value, format is: # -s, --long ARGS else: default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - return ', '.join(action.option_strings) + ' ' + args_string + option_strings = color_option_strings(action.option_strings) + args_string = ( + f"{t.label}{self._format_args(action, default)}{t.reset}" + ) + return ', '.join(option_strings) + ' ' + args_string def _metavar_formatter(self, action, default_metavar): if action.metavar is not None: @@ -590,16 +663,19 @@ def _format_args(self, action, default_metavar): return result def _expand_help(self, action): + help_string = self._get_help_string(action) + if '%' not in help_string: + return help_string params = dict(vars(action), prog=self._prog) for name in list(params): - if params[name] is SUPPRESS: + value = params[name] + if value is SUPPRESS: del params[name] - for name in list(params): - if hasattr(params[name], '__name__'): - params[name] = params[name].__name__ + elif hasattr(value, '__name__'): + params[name] = value.__name__ if params.get('choices') is not None: params['choices'] = ', '.join(map(str, params['choices'])) - return self._get_help_string(action) % params + return help_string % params def _iter_indented_subactions(self, action): try: @@ -844,22 +920,16 @@ def format_usage(self): return self.option_strings[0] def __call__(self, parser, namespace, values, option_string=None): - raise NotImplementedError(_('.__call__() not defined')) - + raise NotImplementedError('.__call__() not defined') -# FIXME: remove together with `BooleanOptionalAction` deprecated arguments. -_deprecated_default = object() class BooleanOptionalAction(Action): def __init__(self, option_strings, dest, default=None, - type=_deprecated_default, - choices=_deprecated_default, required=False, help=None, - metavar=_deprecated_default, deprecated=False): _option_strings = [] @@ -867,38 +937,19 @@ def __init__(self, _option_strings.append(option_string) if option_string.startswith('--'): + if option_string.startswith('--no-'): + raise ValueError(f'invalid option name {option_string!r} ' + f'for BooleanOptionalAction') option_string = '--no-' + option_string[2:] _option_strings.append(option_string) - # We need `_deprecated` special value to ban explicit arguments that - # match default value. Like: - # parser.add_argument('-f', action=BooleanOptionalAction, type=int) - for field_name in ('type', 'choices', 'metavar'): - if locals()[field_name] is not _deprecated_default: - import warnings - warnings._deprecated( - field_name, - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - if type is _deprecated_default: - type = None - if choices is _deprecated_default: - choices = None - if metavar is _deprecated_default: - metavar = None - super().__init__( option_strings=_option_strings, dest=dest, nargs=0, default=default, - type=type, - choices=choices, required=required, help=help, - metavar=metavar, deprecated=deprecated) @@ -1180,6 +1231,7 @@ def __init__(self, self._name_parser_map = {} self._choices_actions = [] self._deprecated = set() + self._color = True super(_SubParsersAction, self).__init__( option_strings=option_strings, @@ -1195,23 +1247,30 @@ def add_parser(self, name, *, deprecated=False, **kwargs): if kwargs.get('prog') is None: kwargs['prog'] = '%s %s' % (self._prog_prefix, name) + # set color + if kwargs.get('color') is None: + kwargs['color'] = self._color + aliases = kwargs.pop('aliases', ()) if name in self._name_parser_map: - raise ArgumentError(self, _('conflicting subparser: %s') % name) + raise ValueError(f'conflicting subparser: {name}') for alias in aliases: if alias in self._name_parser_map: - raise ArgumentError( - self, _('conflicting subparser alias: %s') % alias) + raise ValueError(f'conflicting subparser alias: {alias}') # create a pseudo-action to hold the choice help if 'help' in kwargs: help = kwargs.pop('help') choice_action = self._ChoicesPseudoAction(name, aliases, help) self._choices_actions.append(choice_action) + else: + choice_action = None # create the parser and add it to the map parser = self._parser_class(**kwargs) + if choice_action is not None: + parser._check_help(choice_action) self._name_parser_map[name] = parser # make parser available under aliases also @@ -1276,7 +1335,7 @@ def __call__(self, parser, namespace, values, option_string=None): # ============== class FileType(object): - """Factory for creating file object types + """Deprecated factory for creating file object types Instances of FileType are typically passed as type= arguments to the ArgumentParser add_argument() method. @@ -1293,6 +1352,12 @@ class FileType(object): """ def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None): + import warnings + warnings.warn( + "FileType is deprecated. Simply open files after parsing arguments.", + category=PendingDeprecationWarning, + stacklevel=2 + ) self._mode = mode self._bufsize = bufsize self._encoding = encoding @@ -1396,7 +1461,7 @@ def __init__(self, self._defaults = {} # determines whether an "option" looks like a negative number - self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') + self._negative_number_matcher = _re.compile(r'-\.?\d') # whether or not there are any optionals that look like negative # numbers -- uses a list so it can be shared and edited @@ -1449,7 +1514,8 @@ def add_argument(self, *args, **kwargs): chars = self.prefix_chars if not args or len(args) == 1 and args[0][0] not in chars: if args and 'dest' in kwargs: - raise ValueError('dest supplied twice for positional argument') + raise TypeError('dest supplied twice for positional argument,' + ' did you mean metavar?') kwargs = self._get_positional_kwargs(*args, **kwargs) # otherwise, we're adding an optional argument @@ -1465,27 +1531,34 @@ def add_argument(self, *args, **kwargs): kwargs['default'] = self.argument_default # create the action object, and add it to the parser + action_name = kwargs.get('action') action_class = self._pop_action_class(kwargs) if not callable(action_class): - raise ValueError('unknown action "%s"' % (action_class,)) + raise ValueError(f'unknown action {action_class!r}') action = action_class(**kwargs) + # raise an error if action for positional argument does not + # consume arguments + if not action.option_strings and action.nargs == 0: + raise ValueError(f'action {action_name!r} is not valid for positional arguments') + # raise an error if the action type is not callable type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): - raise ValueError('%r is not callable' % (type_func,)) + raise TypeError(f'{type_func!r} is not callable') if type_func is FileType: - raise ValueError('%r is a FileType class object, instance of it' - ' must be passed' % (type_func,)) + raise TypeError(f'{type_func!r} is a FileType class object, ' + f'instance of it must be passed') # raise an error if the metavar does not match the type if hasattr(self, "_get_formatter"): + formatter = self._get_formatter() try: - self._get_formatter()._format_args(action, None) + formatter._format_args(action, None) except TypeError: raise ValueError("length of metavar tuple does not match nargs") - + self._check_help(action) return self._add_action(action) def add_argument_group(self, *args, **kwargs): @@ -1529,8 +1602,8 @@ def _add_container_actions(self, container): if group.title in title_group_map: # This branch could happen if a derived class added # groups with duplicated titles in __init__ - msg = _('cannot merge actions - two groups are named %r') - raise ValueError(msg % (group.title)) + msg = f'cannot merge actions - two groups are named {group.title!r}' + raise ValueError(msg) title_group_map[group.title] = group # map each action to its group @@ -1571,13 +1644,15 @@ def _add_container_actions(self, container): def _get_positional_kwargs(self, dest, **kwargs): # make sure required is not specified if 'required' in kwargs: - msg = _("'required' is an invalid argument for positionals") + msg = "'required' is an invalid argument for positionals" raise TypeError(msg) # mark positional arguments as required if at least one is # always required nargs = kwargs.get('nargs') - if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS, 0]: + if nargs == 0: + raise ValueError('nargs for positionals must be != 0') + if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS]: kwargs['required'] = True # return the keyword arguments with no option strings @@ -1590,11 +1665,9 @@ def _get_optional_kwargs(self, *args, **kwargs): for option_string in args: # error on strings that don't start with an appropriate prefix if not option_string[0] in self.prefix_chars: - args = {'option': option_string, - 'prefix_chars': self.prefix_chars} - msg = _('invalid option string %(option)r: ' - 'must start with a character %(prefix_chars)r') - raise ValueError(msg % args) + raise ValueError( + f'invalid option string {option_string!r}: ' + f'must start with a character {self.prefix_chars!r}') # strings starting with two prefix characters are long options option_strings.append(option_string) @@ -1610,8 +1683,8 @@ def _get_optional_kwargs(self, *args, **kwargs): dest_option_string = option_strings[0] dest = dest_option_string.lstrip(self.prefix_chars) if not dest: - msg = _('dest= is required for options like %r') - raise ValueError(msg % option_string) + msg = f'dest= is required for options like {option_string!r}' + raise TypeError(msg) dest = dest.replace('-', '_') # return the updated keyword arguments @@ -1627,8 +1700,8 @@ def _get_handler(self): try: return getattr(self, handler_func_name) except AttributeError: - msg = _('invalid conflict_resolution value: %r') - raise ValueError(msg % self.conflict_handler) + msg = f'invalid conflict_resolution value: {self.conflict_handler!r}' + raise ValueError(msg) def _check_conflict(self, action): @@ -1667,10 +1740,26 @@ def _handle_conflict_resolve(self, action, conflicting_actions): if not action.option_strings: action.container._remove_action(action) + def _check_help(self, action): + if action.help and hasattr(self, "_get_formatter"): + formatter = self._get_formatter() + try: + formatter._expand_help(action) + except (ValueError, TypeError, KeyError) as exc: + raise ValueError('badly formed help string') from exc + class _ArgumentGroup(_ActionsContainer): def __init__(self, container, title=None, description=None, **kwargs): + if 'prefix_chars' in kwargs: + import warnings + depr_msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + warnings.warn(depr_msg, DeprecationWarning, stacklevel=3) + # add any missing keyword arguments by checking the container update = kwargs.setdefault update('conflict_handler', container.conflict_handler) @@ -1702,14 +1791,7 @@ def _remove_action(self, action): self._group_actions.remove(action) def add_argument_group(self, *args, **kwargs): - import warnings - warnings.warn( - "Nesting argument groups is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - return super().add_argument_group(*args, **kwargs) - + raise ValueError('argument groups cannot be nested') class _MutuallyExclusiveGroup(_ArgumentGroup): @@ -1720,7 +1802,7 @@ def __init__(self, container, required=False): def _add_action(self, action): if action.required: - msg = _('mutually exclusive arguments must be optional') + msg = 'mutually exclusive arguments must be optional' raise ValueError(msg) action = self._container._add_action(action) self._group_actions.append(action) @@ -1730,14 +1812,29 @@ def _remove_action(self, action): self._container._remove_action(action) self._group_actions.remove(action) - def add_mutually_exclusive_group(self, *args, **kwargs): - import warnings - warnings.warn( - "Nesting mutually exclusive groups is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - return super().add_mutually_exclusive_group(*args, **kwargs) + def add_mutually_exclusive_group(self, **kwargs): + raise ValueError('mutually exclusive groups cannot be nested') + +def _prog_name(prog=None): + if prog is not None: + return prog + arg0 = _sys.argv[0] + try: + modspec = _sys.modules['__main__'].__spec__ + except (KeyError, AttributeError): + # possibly PYTHONSTARTUP or -X presite or other weird edge case + # no good answer here, so fall back to the default + modspec = None + if modspec is None: + # simple script + return _os.path.basename(arg0) + py = _os.path.basename(_sys.executable) + if modspec.name != '__main__': + # imported module or package + modname = modspec.name.removesuffix('.__main__') + return f'{py} -m {modname}' + # directory or ZIP file + return f'{py} {arg0}' class ArgumentParser(_AttributeHolder, _ActionsContainer): @@ -1760,6 +1857,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer): - allow_abbrev -- Allow long options to be abbreviated unambiguously - exit_on_error -- Determines whether or not ArgumentParser exits with error info when an error occurs + - suggest_on_error - Enables suggestions for mistyped argument choices + and subparser names (default: ``False``) + - color - Allow color output in help messages (default: ``False``) """ def __init__(self, @@ -1775,19 +1875,18 @@ def __init__(self, conflict_handler='error', add_help=True, allow_abbrev=True, - exit_on_error=True): - + exit_on_error=True, + *, + suggest_on_error=False, + color=True, + ): superinit = super(ArgumentParser, self).__init__ superinit(description=description, prefix_chars=prefix_chars, argument_default=argument_default, conflict_handler=conflict_handler) - # default setting for prog - if prog is None: - prog = _os.path.basename(_sys.argv[0]) - - self.prog = prog + self.prog = _prog_name(prog) self.usage = usage self.epilog = epilog self.formatter_class = formatter_class @@ -1795,6 +1894,8 @@ def __init__(self, self.add_help = add_help self.allow_abbrev = allow_abbrev self.exit_on_error = exit_on_error + self.suggest_on_error = suggest_on_error + self.color = color add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) @@ -1844,7 +1945,7 @@ def _get_kwargs(self): def add_subparsers(self, **kwargs): if self._subparsers is not None: - raise ArgumentError(None, _('cannot have multiple subparser arguments')) + raise ValueError('cannot have multiple subparser arguments') # add the parser class to the arguments if it's not present kwargs.setdefault('parser_class', type(self)) @@ -1859,15 +1960,19 @@ def add_subparsers(self, **kwargs): # prog defaults to the usage message of this parser, skipping # optional arguments and with no "usage:" prefix if kwargs.get('prog') is None: - formatter = self._get_formatter() + # Create formatter without color to avoid storing ANSI codes in prog + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(False) positionals = self._get_positional_actions() groups = self._mutually_exclusive_groups - formatter.add_usage(self.usage, positionals, groups, '') + formatter.add_usage(None, positionals, groups, '') kwargs['prog'] = formatter.format_help().strip() # create the parsers action and add it to the positionals list parsers_class = self._pop_action_class(kwargs, 'parsers') action = parsers_class(option_strings=[], **kwargs) + action._color = self.color + self._check_help(action) self._subparsers._add_action(action) # return the created parsers action @@ -2498,7 +2603,6 @@ def _get_values(self, action, arg_strings): value = action.default if isinstance(value, str) and value is not SUPPRESS: value = self._get_value(action, value) - self._check_value(action, value) # when nargs='*' on a positional, if there were no command-line # args, use the default if it is anything other than None @@ -2506,11 +2610,8 @@ def _get_values(self, action, arg_strings): not action.option_strings): if action.default is not None: value = action.default - self._check_value(action, value) else: - # since arg_strings is always [] at this point - # there is no need to use self._check_value(action, value) - value = arg_strings + value = [] # single argument or optional argument produces a single value elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: @@ -2543,8 +2644,7 @@ def _get_values(self, action, arg_strings): def _get_value(self, action, arg_string): type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): - msg = _('%r is not callable') - raise ArgumentError(action, msg % type_func) + raise TypeError(f'{type_func!r} is not callable') # convert the value to the appropriate type try: @@ -2568,14 +2668,27 @@ def _get_value(self, action, arg_string): def _check_value(self, action, value): # converted value must be one of the choices (if specified) choices = action.choices - if choices is not None: - if isinstance(choices, str): - choices = iter(choices) - if value not in choices: - args = {'value': str(value), - 'choices': ', '.join(map(str, action.choices))} - msg = _('invalid choice: %(value)r (choose from %(choices)s)') - raise ArgumentError(action, msg % args) + if choices is None: + return + + if isinstance(choices, str): + choices = iter(choices) + + if value not in choices: + args = {'value': str(value), + 'choices': ', '.join(map(str, action.choices))} + msg = _('invalid choice: %(value)r (choose from %(choices)s)') + + if self.suggest_on_error and isinstance(value, str): + if all(isinstance(choice, str) for choice in action.choices): + import difflib + suggestions = difflib.get_close_matches(value, action.choices, 1) + if suggestions: + args['closest'] = suggestions[0] + msg = _('invalid choice: %(value)r, maybe you meant %(closest)r? ' + '(choose from %(choices)s)') + + raise ArgumentError(action, msg % args) # ======================= # Help-formatting methods @@ -2611,7 +2724,9 @@ def format_help(self): return formatter.format_help() def _get_formatter(self): - return self.formatter_class(prog=self.prog) + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(self.color) + return formatter # ===================== # Help-printing methods diff --git a/Lib/calendar.py b/Lib/calendar.py index 8c1c646da46..18f76d52ff8 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -428,6 +428,7 @@ def formatyear(self, theyear, w=2, l=1, c=6, m=3): headers = (header for k in months) a(formatstring(headers, colwidth, c).rstrip()) a('\n'*l) + # max number of weeks for this row height = max(len(cal) for cal in row) for j in range(height): @@ -646,6 +647,117 @@ def formatmonthname(self, theyear, themonth, withyear=True): with different_locale(self.locale): return super().formatmonthname(theyear, themonth, withyear) + +class _CLIDemoCalendar(TextCalendar): + def __init__(self, highlight_day=None, *args, **kwargs): + super().__init__(*args, **kwargs) + self.highlight_day = highlight_day + + def formatweek(self, theweek, width, *, highlight_day=None): + """ + Returns a single week in a string (no newline). + """ + if highlight_day: + from _colorize import get_colors + + ansi = get_colors() + highlight = f"{ansi.BLACK}{ansi.BACKGROUND_YELLOW}" + reset = ansi.RESET + else: + highlight = reset = "" + + return ' '.join( + ( + f"{highlight}{self.formatday(d, wd, width)}{reset}" + if d == highlight_day + else self.formatday(d, wd, width) + ) + for (d, wd) in theweek + ) + + def formatmonth(self, theyear, themonth, w=0, l=0): + """ + Return a month's calendar string (multi-line). + """ + if ( + self.highlight_day + and self.highlight_day.year == theyear + and self.highlight_day.month == themonth + ): + highlight_day = self.highlight_day.day + else: + highlight_day = None + w = max(2, w) + l = max(1, l) + s = self.formatmonthname(theyear, themonth, 7 * (w + 1) - 1) + s = s.rstrip() + s += '\n' * l + s += self.formatweekheader(w).rstrip() + s += '\n' * l + for week in self.monthdays2calendar(theyear, themonth): + s += self.formatweek(week, w, highlight_day=highlight_day).rstrip() + s += '\n' * l + return s + + def formatyear(self, theyear, w=2, l=1, c=6, m=3): + """ + Returns a year's calendar as a multi-line string. + """ + w = max(2, w) + l = max(1, l) + c = max(2, c) + colwidth = (w + 1) * 7 - 1 + v = [] + a = v.append + a(repr(theyear).center(colwidth*m+c*(m-1)).rstrip()) + a('\n'*l) + header = self.formatweekheader(w) + for (i, row) in enumerate(self.yeardays2calendar(theyear, m)): + # months in this row + months = range(m*i+1, min(m*(i+1)+1, 13)) + a('\n'*l) + names = (self.formatmonthname(theyear, k, colwidth, False) + for k in months) + a(formatstring(names, colwidth, c).rstrip()) + a('\n'*l) + headers = (header for k in months) + a(formatstring(headers, colwidth, c).rstrip()) + a('\n'*l) + + if ( + self.highlight_day + and self.highlight_day.year == theyear + and self.highlight_day.month in months + ): + month_pos = months.index(self.highlight_day.month) + else: + month_pos = None + + # max number of weeks for this row + height = max(len(cal) for cal in row) + for j in range(height): + weeks = [] + for k, cal in enumerate(row): + if j >= len(cal): + weeks.append('') + else: + day = ( + self.highlight_day.day if k == month_pos else None + ) + weeks.append( + self.formatweek(cal[j], w, highlight_day=day) + ) + a(formatstring(weeks, colwidth, c).rstrip()) + a('\n' * l) + return ''.join(v) + + +class _CLIDemoLocaleCalendar(LocaleTextCalendar, _CLIDemoCalendar): + def __init__(self, highlight_day=None, *args, **kwargs): + super().__init__(*args, **kwargs) + self.highlight_day = highlight_day + + # Support for old module level interface c = TextCalendar() @@ -698,7 +810,7 @@ def timegm(tuple): def main(args=None): import argparse - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) textgroup = parser.add_argument_group('text only arguments') htmlgroup = parser.add_argument_group('html only arguments') textgroup.add_argument( @@ -765,6 +877,7 @@ def main(args=None): sys.exit(1) locale = options.locale, options.encoding + today = datetime.date.today() if options.type == "html": if options.month: @@ -781,23 +894,23 @@ def main(args=None): optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write if options.year is None: - write(cal.formatyearpage(datetime.date.today().year, **optdict)) + write(cal.formatyearpage(today.year, **optdict)) else: write(cal.formatyearpage(options.year, **optdict)) else: if options.locale: - cal = LocaleTextCalendar(locale=locale) + cal = _CLIDemoLocaleCalendar(highlight_day=today, locale=locale) else: - cal = TextCalendar() + cal = _CLIDemoCalendar(highlight_day=today) cal.setfirstweekday(options.first_weekday) optdict = dict(w=options.width, l=options.lines) if options.month is None: optdict["c"] = options.spacing optdict["m"] = options.months - if options.month is not None: + else: _validate_month(options.month) if options.year is None: - result = cal.formatyear(datetime.date.today().year, **optdict) + result = cal.formatyear(today.year, **optdict) elif options.month is None: result = cal.formatyear(options.year, **optdict) else: diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index ab6d32478e4..21bbfad0fe6 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "25.2" +_PIP_VERSION = "25.3" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora @@ -205,7 +205,7 @@ def _uninstall_helper(*, verbosity=0): def _main(argv=None): import argparse - parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser = argparse.ArgumentParser(color=True) parser.add_argument( "--version", action="version", diff --git a/Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl similarity index 78% rename from Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl index 4db7e720718..755e1aa0c3d 100644 Binary files a/Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl differ diff --git a/Lib/ensurepip/_uninstall.py b/Lib/ensurepip/_uninstall.py index b257904328d..4183c28a809 100644 --- a/Lib/ensurepip/_uninstall.py +++ b/Lib/ensurepip/_uninstall.py @@ -6,7 +6,7 @@ def _main(argv=None): - parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", diff --git a/Lib/inspect.py b/Lib/inspect.py index 5a814f97b5b..d2a04546959 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -220,13 +220,26 @@ def get_annotations(obj, *, globals=None, locals=None, eval_str=False): """ if isinstance(obj, type): # class - obj_dict = getattr(obj, '__dict__', None) - if obj_dict and hasattr(obj_dict, 'get'): - ann = obj_dict.get('__annotations__', None) - if isinstance(ann, types.GetSetDescriptorType): - ann = None - else: + # XXX: RUSTPYTHON + # PEP 649: Use descriptor access to get __annotations__ + # The __annotations__ attribute is now provided via a descriptor + # that calls __annotate__ to compute the annotations lazily + try: + ann = obj.__annotations__ + except AttributeError: ann = None + # Ensure we got this class's own annotations, not inherited + if ann is not None: + # Check if __annotations__ is in this class's __dict__ + # or is computed by a descriptor (both are valid) + obj_dict = getattr(obj, '__dict__', None) + if obj_dict is not None: + dict_ann = obj_dict.get('__annotations__', None) + # If it's a descriptor or cached dict, use what we got from attribute access + # If it's inherited (not in __dict__ at all and no __annotate__), return empty + # XXX: RUSTPYTHON - also check __annotate_func__ (PEP 649 descriptor) + if dict_ann is None and '__annotate__' not in obj_dict and '__annotations_cache__' not in obj_dict and not hasattr(obj, '__annotate_func__'): + ann = None obj_globals = None module_name = getattr(obj, '__module__', None) diff --git a/Lib/site.py b/Lib/site.py index 2983ca71544..5305d67b3b8 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -73,7 +73,7 @@ import os import builtins import _sitebuiltins -import io +import _io as io import stat import errno @@ -95,6 +95,12 @@ def _trace(message): print(message, file=sys.stderr) +def _warn(*args, **kwargs): + import warnings + + warnings.warn(*args, **kwargs) + + def makepath(*paths): dir = os.path.join(*paths) try: @@ -444,9 +450,9 @@ def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) builtins.credits = _sitebuiltins._Printer("credits", """\ - Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software - Foundation, and a cast of thousands for supporting Python - development. See www.python.org for more information.""") +Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software +Foundation, and a cast of thousands for supporting Python +development. See www.python.org for more information.""") files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. @@ -574,7 +580,7 @@ def register_readline(): def write_history(): try: readline_module.write_history_file(history) - except (FileNotFoundError, PermissionError): + except FileNotFoundError, PermissionError: # home directory does not exist or is not writable # https://bugs.python.org/issue19891 pass @@ -626,17 +632,17 @@ def venv(known_paths): elif key == 'home': sys._home = value - sys.prefix = sys.exec_prefix = site_prefix + if sys.prefix != site_prefix: + _warn(f'Unexpected value in sys.prefix, expected {site_prefix}, got {sys.prefix}', RuntimeWarning) + if sys.exec_prefix != site_prefix: + _warn(f'Unexpected value in sys.exec_prefix, expected {site_prefix}, got {sys.exec_prefix}', RuntimeWarning) # Doing this here ensures venv takes precedence over user-site addsitepackages(known_paths, [sys.prefix]) - # addsitepackages will process site_prefix again if its in PREFIXES, - # but that's ok; known_paths will prevent anything being added twice if system_site == "true": - PREFIXES.insert(0, sys.prefix) + PREFIXES += [sys.base_prefix, sys.base_exec_prefix] else: - PREFIXES = [sys.prefix] ENABLE_USER_SITE = False return known_paths @@ -646,7 +652,7 @@ def execsitecustomize(): """Run custom site specific code, if available.""" try: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError as exc: if exc.name == 'sitecustomize': pass @@ -666,7 +672,7 @@ def execusercustomize(): """Run custom user specific code, if available.""" try: try: - import usercustomize + import usercustomize # noqa: F401 except ImportError as exc: if exc.name == 'usercustomize': pass diff --git a/Lib/string.py b/Lib/string/__init__.py similarity index 87% rename from Lib/string.py rename to Lib/string/__init__.py index 2eab6d4f595..eab5067c9b1 100644 --- a/Lib/string.py +++ b/Lib/string/__init__.py @@ -49,11 +49,18 @@ def capwords(s, sep=None): #################################################################### -import re as _re -from collections import ChainMap as _ChainMap - _sentinel_dict = {} + +class _TemplatePattern: + # This descriptor is overwritten in ``Template._compile_pattern()``. + def __get__(self, instance, cls=None): + if cls is None: + return self + return cls._compile_pattern() +_TemplatePattern = _TemplatePattern() + + class Template: """A string class for supporting $-substitutions.""" @@ -64,14 +71,21 @@ class Template: # See https://bugs.python.org/issue31672 idpattern = r'(?a:[_a-z][_a-z0-9]*)' braceidpattern = None - flags = _re.IGNORECASE + flags = None # default: re.IGNORECASE + + pattern = _TemplatePattern # use a descriptor to compile the pattern def __init_subclass__(cls): super().__init_subclass__() - if 'pattern' in cls.__dict__: - pattern = cls.pattern - else: - delim = _re.escape(cls.delimiter) + cls._compile_pattern() + + @classmethod + def _compile_pattern(cls): + import re # deferred import, for performance + + pattern = cls.__dict__.get('pattern', _TemplatePattern) + if pattern is _TemplatePattern: + delim = re.escape(cls.delimiter) id = cls.idpattern bid = cls.braceidpattern or cls.idpattern pattern = fr""" @@ -82,7 +96,10 @@ def __init_subclass__(cls): (?P) # Other ill-formed delimiter exprs ) """ - cls.pattern = _re.compile(pattern, cls.flags | _re.VERBOSE) + if cls.flags is None: + cls.flags = re.IGNORECASE + pat = cls.pattern = re.compile(pattern, cls.flags | re.VERBOSE) + return pat def __init__(self, template): self.template = template @@ -105,7 +122,8 @@ def substitute(self, mapping=_sentinel_dict, /, **kws): if mapping is _sentinel_dict: mapping = kws elif kws: - mapping = _ChainMap(kws, mapping) + from collections import ChainMap + mapping = ChainMap(kws, mapping) # Helper function for .sub() def convert(mo): # Check the most common path first. @@ -124,7 +142,8 @@ def safe_substitute(self, mapping=_sentinel_dict, /, **kws): if mapping is _sentinel_dict: mapping = kws elif kws: - mapping = _ChainMap(kws, mapping) + from collections import ChainMap + mapping = ChainMap(kws, mapping) # Helper function for .sub() def convert(mo): named = mo.group('named') or mo.group('braced') @@ -170,10 +189,6 @@ def get_identifiers(self): self.pattern) return ids -# Initialize Template.pattern. __init_subclass__() is automatically called -# only for subclasses, not for the Template class itself. -Template.__init_subclass__() - ######################################################################## # the Formatter class @@ -212,19 +227,20 @@ def _vformat(self, format_string, args, kwargs, used_args, recursion_depth, # this is some markup, find the object and do # the formatting - # handle arg indexing when empty field_names are given. - if field_name == '': + # handle arg indexing when empty field first parts are given. + field_first, _ = _string.formatter_field_name_split(field_name) + if field_first == '': if auto_arg_index is False: raise ValueError('cannot switch from manual field ' 'specification to automatic field ' 'numbering') - field_name = str(auto_arg_index) + field_name = str(auto_arg_index) + field_name auto_arg_index += 1 - elif field_name.isdigit(): + elif isinstance(field_first, int): if auto_arg_index: - raise ValueError('cannot switch from manual field ' - 'specification to automatic field ' - 'numbering') + raise ValueError('cannot switch from automatic field ' + 'numbering to manual field ' + 'specification') # disable auto arg incrementing, if it gets # used later on, then an exception will be raised auto_arg_index = False diff --git a/Lib/string/templatelib.py b/Lib/string/templatelib.py new file mode 100644 index 00000000000..8164872432a --- /dev/null +++ b/Lib/string/templatelib.py @@ -0,0 +1,33 @@ +"""Support for template string literals (t-strings).""" + +t = t"{0}" +Template = type(t) +Interpolation = type(t.interpolations[0]) +del t + +def convert(obj, /, conversion): + """Convert *obj* using formatted string literal semantics.""" + if conversion is None: + return obj + if conversion == 'r': + return repr(obj) + if conversion == 's': + return str(obj) + if conversion == 'a': + return ascii(obj) + raise ValueError(f'invalid conversion specifier: {conversion}') + +def _template_unpickle(*args): + import itertools + + if len(args) != 2: + raise ValueError('Template expects tuple of length 2 to unpickle') + + strings, interpolations = args + parts = [] + for string, interpolation in itertools.zip_longest(strings, interpolations): + if string is not None: + parts.append(string) + if interpolation is not None: + parts.append(interpolation) + return Template(*parts) diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py index b2f0bb1386f..31dc60fec35 100644 --- a/Lib/test/test__colorize.py +++ b/Lib/test/test__colorize.py @@ -1,4 +1,5 @@ import contextlib +import dataclasses import io import sys import unittest @@ -21,6 +22,42 @@ def supports_virtual_terminal(): return contextlib.nullcontext() +class TestTheme(unittest.TestCase): + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_attributes(self): + # only theme configurations attributes by default + for field in dataclasses.fields(_colorize.Theme): + with self.subTest(field.name): + self.assertIsSubclass(field.type, _colorize.ThemeSection) + self.assertIsNotNone(field.default_factory) + + def test_copy_with(self): + theme = _colorize.Theme() + + copy = theme.copy_with() + self.assertEqual(theme, copy) + + unittest_no_colors = _colorize.Unittest.no_colors() + copy = theme.copy_with(unittest=unittest_no_colors) + self.assertEqual(copy.argparse, theme.argparse) + self.assertEqual(copy.syntax, theme.syntax) + self.assertEqual(copy.traceback, theme.traceback) + self.assertEqual(copy.unittest, unittest_no_colors) + + def test_no_colors(self): + # idempotence test + theme_no_colors = _colorize.Theme().no_colors() + theme_no_colors_no_colors = theme_no_colors.no_colors() + self.assertEqual(theme_no_colors, theme_no_colors_no_colors) + + # attributes check + for section in dataclasses.fields(_colorize.Theme): + with self.subTest(section.name): + section_theme = getattr(theme_no_colors, section.name) + self.assertEqual(section_theme, section.type.no_colors()) + + class TestColorizeFunction(unittest.TestCase): def test_colorized_detection_checks_for_environment_variables(self): def check(env, fallback, expected): @@ -129,6 +166,17 @@ def test_colorized_detection_checks_for_file(self): file.isatty.return_value = False self.assertEqual(_colorize.can_colorize(file=file), False) + # The documentation for file.fileno says: + # > An OSError is raised if the IO object does not use a file descriptor. + # gh-141570: Check OSError is caught and handled + with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError): + file = unittest.mock.MagicMock() + file.fileno.side_effect = OSError + file.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), True) + file.isatty.return_value = False + self.assertEqual(_colorize.can_colorize(file=file), False) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index b7e995334fe..beda61be8a4 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1,11 +1,13 @@ # Author: Steven J. Bethard . +import _colorize import contextlib import functools import inspect import io import operator import os +import py_compile import shutil import stat import sys @@ -16,12 +18,22 @@ import warnings from enum import StrEnum -from test.support import captured_stderr +from test.support import ( + captured_stderr, + force_not_colorized, + force_not_colorized_test_class, + swap_attr, +) +from test.support import import_helper from test.support import os_helper +from test.support import script_helper from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots from unittest import mock +py = os.path.basename(sys.executable) + + class StdIOBuffer(io.TextIOWrapper): '''Replacement for writable io.StringIO that behaves more like real file @@ -631,9 +643,9 @@ class TestOptionalsNargsOptional(ParserTestCase): Sig('-w', nargs='?'), Sig('-x', nargs='?', const=42), Sig('-y', nargs='?', default='spam'), - Sig('-z', nargs='?', type=int, const='42', default='84'), + Sig('-z', nargs='?', type=int, const='42', default='84', choices=[1, 2]), ] - failures = ['2'] + failures = ['2', '-z a', '-z 42', '-z 84'] successes = [ ('', NS(w=None, x=None, y='spam', z=84)), ('-w', NS(w=None, x=None, y='spam', z=84)), @@ -777,48 +789,12 @@ def test_const(self): self.assertIn("got an unexpected keyword argument 'const'", str(cm.exception)) - def test_deprecated_init_kw(self): - # See gh-92248 + def test_invalid_name(self): parser = argparse.ArgumentParser() - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-a', - action=argparse.BooleanOptionalAction, - type=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-b', - action=argparse.BooleanOptionalAction, - type=bool, - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-c', - action=argparse.BooleanOptionalAction, - metavar=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-d', - action=argparse.BooleanOptionalAction, - metavar='d', - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-e', - action=argparse.BooleanOptionalAction, - choices=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-f', - action=argparse.BooleanOptionalAction, - choices=(), - ) + with self.assertRaises(ValueError) as cm: + parser.add_argument('--no-foo', action=argparse.BooleanOptionalAction) + self.assertEqual(str(cm.exception), + "invalid option name '--no-foo' for BooleanOptionalAction") class TestBooleanOptionalActionRequired(ParserTestCase): """Tests BooleanOptionalAction required""" @@ -1036,6 +1012,7 @@ def test_parse_enum_value(self): args = parser.parse_args(['--color', 'red']) self.assertEqual(args.color, self.Color.RED) + @force_not_colorized def test_help_message_contains_enum_choices(self): parser = argparse.ArgumentParser() parser.add_argument('--color', choices=self.Color, help='Choose a color') @@ -1101,8 +1078,8 @@ class TestPositionalsNargsZeroOrMore(ParserTestCase): class TestPositionalsNargsZeroOrMoreDefault(ParserTestCase): """Test a Positional that specifies unlimited nargs and a default""" - argument_signatures = [Sig('foo', nargs='*', default='bar')] - failures = ['-x'] + argument_signatures = [Sig('foo', nargs='*', default='bar', choices=['a', 'b'])] + failures = ['-x', 'bar', 'a c'] successes = [ ('', NS(foo='bar')), ('a', NS(foo=['a'])), @@ -1135,8 +1112,8 @@ class TestPositionalsNargsOptional(ParserTestCase): class TestPositionalsNargsOptionalDefault(ParserTestCase): """Tests an Optional Positional with a default value""" - argument_signatures = [Sig('foo', nargs='?', default=42)] - failures = ['-x', 'a b'] + argument_signatures = [Sig('foo', nargs='?', default=42, choices=['a', 'b'])] + failures = ['-x', 'a b', '42'] successes = [ ('', NS(foo=42)), ('a', NS(foo='a')), @@ -1149,9 +1126,9 @@ class TestPositionalsNargsOptionalConvertedDefault(ParserTestCase): """ argument_signatures = [ - Sig('foo', nargs='?', type=int, default='42'), + Sig('foo', nargs='?', type=int, default='42', choices=[1, 2]), ] - failures = ['-x', 'a b', '1 2'] + failures = ['-x', 'a b', '1 2', '42'] successes = [ ('', NS(foo=42)), ('1', NS(foo=1)), @@ -1811,27 +1788,43 @@ def convert_arg_line_to_args(self, arg_line): # Type conversion tests # ===================== +def FileType(*args, **kwargs): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'FileType is deprecated', + PendingDeprecationWarning, __name__) + return argparse.FileType(*args, **kwargs) + + +class TestFileTypeDeprecation(TestCase): + + def test(self): + with self.assertWarns(PendingDeprecationWarning) as cm: + argparse.FileType() + self.assertIn('FileType is deprecated', str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + class TestFileTypeRepr(TestCase): def test_r(self): - type = argparse.FileType('r') + type = FileType('r') self.assertEqual("FileType('r')", repr(type)) def test_wb_1(self): - type = argparse.FileType('wb', 1) + type = FileType('wb', 1) self.assertEqual("FileType('wb', 1)", repr(type)) def test_r_latin(self): - type = argparse.FileType('r', encoding='latin_1') + type = FileType('r', encoding='latin_1') self.assertEqual("FileType('r', encoding='latin_1')", repr(type)) def test_w_big5_ignore(self): - type = argparse.FileType('w', encoding='big5', errors='ignore') + type = FileType('w', encoding='big5', errors='ignore') self.assertEqual("FileType('w', encoding='big5', errors='ignore')", repr(type)) def test_r_1_replace(self): - type = argparse.FileType('r', 1, errors='replace') + type = FileType('r', 1, errors='replace') self.assertEqual("FileType('r', 1, errors='replace')", repr(type)) @@ -1885,7 +1878,6 @@ def __eq__(self, other): text = text.decode('ascii') return self.name == other.name == text - class TestFileTypeR(TempDirMixin, ParserTestCase): """Test the FileType option/argument type for reading files""" @@ -1898,8 +1890,8 @@ def setUp(self): self.create_readonly_file('readonly') argument_signatures = [ - Sig('-x', type=argparse.FileType()), - Sig('spam', type=argparse.FileType('r')), + Sig('-x', type=FileType()), + Sig('spam', type=FileType('r')), ] failures = ['-x', '', 'non-existent-file.txt'] successes = [ @@ -1919,7 +1911,7 @@ def setUp(self): file.close() argument_signatures = [ - Sig('-c', type=argparse.FileType('r'), default='no-file.txt'), + Sig('-c', type=FileType('r'), default='no-file.txt'), ] # should provoke no such file error failures = [''] @@ -1938,8 +1930,8 @@ def setUp(self): file.write(file_name) argument_signatures = [ - Sig('-x', type=argparse.FileType('rb')), - Sig('spam', type=argparse.FileType('rb')), + Sig('-x', type=FileType('rb')), + Sig('spam', type=FileType('rb')), ] failures = ['-x', ''] successes = [ @@ -1977,8 +1969,8 @@ def setUp(self): self.create_writable_file('writable') argument_signatures = [ - Sig('-x', type=argparse.FileType('w')), - Sig('spam', type=argparse.FileType('w')), + Sig('-x', type=FileType('w')), + Sig('spam', type=FileType('w')), ] failures = ['-x', '', 'readonly'] successes = [ @@ -2000,8 +1992,8 @@ def setUp(self): self.create_writable_file('writable') argument_signatures = [ - Sig('-x', type=argparse.FileType('x')), - Sig('spam', type=argparse.FileType('x')), + Sig('-x', type=FileType('x')), + Sig('spam', type=FileType('x')), ] failures = ['-x', '', 'readonly', 'writable'] successes = [ @@ -2015,8 +2007,8 @@ class TestFileTypeWB(TempDirMixin, ParserTestCase): """Test the FileType option/argument type for writing binary files""" argument_signatures = [ - Sig('-x', type=argparse.FileType('wb')), - Sig('spam', type=argparse.FileType('wb')), + Sig('-x', type=FileType('wb')), + Sig('spam', type=FileType('wb')), ] failures = ['-x', ''] successes = [ @@ -2032,8 +2024,8 @@ class TestFileTypeXB(TestFileTypeX): "Test the FileType option/argument type for writing new binary files only" argument_signatures = [ - Sig('-x', type=argparse.FileType('xb')), - Sig('spam', type=argparse.FileType('xb')), + Sig('-x', type=FileType('xb')), + Sig('spam', type=FileType('xb')), ] successes = [ ('-x foo bar', NS(x=WFile('foo'), spam=WFile('bar'))), @@ -2045,7 +2037,7 @@ class TestFileTypeOpenArgs(TestCase): """Test that open (the builtin) is correctly called""" def test_open_args(self): - FT = argparse.FileType + FT = FileType cases = [ (FT('rb'), ('rb', -1, None, None)), (FT('w', 1), ('w', 1, None, None)), @@ -2060,7 +2052,7 @@ def test_open_args(self): def test_invalid_file_type(self): with self.assertRaises(ValueError): - argparse.FileType('b')('-test') + FileType('b')('-test') class TestFileTypeMissingInitialization(TestCase): @@ -2071,7 +2063,7 @@ class TestFileTypeMissingInitialization(TestCase): def test(self): parser = argparse.ArgumentParser() - with self.assertRaises(ValueError) as cm: + with self.assertRaises(TypeError) as cm: parser.add_argument('-x', type=argparse.FileType) self.assertEqual( @@ -2256,6 +2248,130 @@ class TestActionExtend(ParserTestCase): ] +class TestNegativeNumber(ParserTestCase): + """Test parsing negative numbers""" + + argument_signatures = [ + Sig('--int', type=int), + Sig('--float', type=float), + Sig('--complex', type=complex), + ] + failures = [ + '--float -_.45', + '--float -1__000.0', + '--float -1.0.0', + '--int -1__000', + '--int -1.0', + '--complex -1__000.0j', + '--complex -1.0jj', + '--complex -_.45j', + ] + successes = [ + ('--int -1000 --float -1000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000 --float -1_000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000_000 --float -1_000_000.0', NS(int=-1000000, float=-1000000.0, complex=None)), + ('--float -1_000.0', NS(int=None, float=-1000.0, complex=None)), + ('--float -1_000_000.0_0', NS(int=None, float=-1000000.0, complex=None)), + ('--float -.5', NS(int=None, float=-0.5, complex=None)), + ('--float -.5_000', NS(int=None, float=-0.5, complex=None)), + ('--float -1e3', NS(int=None, float=-1000, complex=None)), + ('--float -1e-3', NS(int=None, float=-0.001, complex=None)), + ('--complex -1j', NS(int=None, float=None, complex=-1j)), + ('--complex -1_000j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1_000.0j', NS(int=None, float=None, complex=-1000.0j)), + ('--complex -1e3j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1e-3j', NS(int=None, float=None, complex=-0.001j)), + ] + +class TestArgumentAndSubparserSuggestions(TestCase): + """Test error handling and suggestion when a user makes a typo""" + + def test_wrong_argument_error_with_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz', maybe you meant 'baz'? (choose from bar, baz)", + excinfo.exception.stderr + ) + + def test_wrong_argument_error_no_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=False) + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_subparsers_with_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('baz',)) + self.assertIn( + "error: argument {foo,bar}: invalid choice: 'baz', maybe you meant" + " 'bar'? (choose from foo, bar)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_subparsers_no_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=False) + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('baz',)) + self.assertIn( + "error: argument {foo,bar}: invalid choice: 'baz' (choose from foo, bar)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_no_suggestion_implicit(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_empty(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[]) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from )", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_int(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[1, 2]) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('3',)) + self.assertIn( + "error: argument foo: invalid choice: '3' (choose from 1, 2)", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_mixed_types(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[1, '2']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('3',)) + self.assertIn( + "error: argument foo: invalid choice: '3' (choose from 1, 2)", + excinfo.exception.stderr, + ) + + class TestInvalidAction(TestCase): """Test invalid user defined Action""" @@ -2269,17 +2385,31 @@ def test_invalid_type(self): self.assertRaises(NotImplementedError, parser.parse_args, ['--foo', 'bar']) def test_modified_invalid_action(self): - parser = ErrorRaisingArgumentParser() + parser = argparse.ArgumentParser(exit_on_error=False) action = parser.add_argument('--foo') # Someone got crazy and did this action.type = 1 - self.assertRaises(ArgumentParserError, parser.parse_args, ['--foo', 'bar']) + self.assertRaisesRegex(TypeError, '1 is not callable', + parser.parse_args, ['--foo', 'bar']) + action.type = () + self.assertRaisesRegex(TypeError, r'\(\) is not callable', + parser.parse_args, ['--foo', 'bar']) + # It is impossible to distinguish a TypeError raised due to a mismatch + # of the required function arguments from a TypeError raised for an incorrect + # argument value, and using the heavy inspection machinery is not worthwhile + # as it does not reliably work in all cases. + # Therefore, a generic ArgumentError is raised to handle this logical error. + action.type = pow + self.assertRaisesRegex(argparse.ArgumentError, + "argument --foo: invalid pow value: 'bar'", + parser.parse_args, ['--foo', 'bar']) # ================ # Subparsers tests # ================ +@force_not_colorized_test_class class TestAddSubparsers(TestCase): """Test the add_subparsers method""" @@ -2287,16 +2417,17 @@ def assertArgumentParserError(self, *args, **kwargs): self.assertRaises(ArgumentParserError, *args, **kwargs) def _get_parser(self, subparser_help=False, prefix_chars=None, - aliases=False): + aliases=False, usage=None): # create a parser with a subparsers argument if prefix_chars: parser = ErrorRaisingArgumentParser( - prog='PROG', description='main description', prefix_chars=prefix_chars) + prog='PROG', description='main description', usage=usage, + prefix_chars=prefix_chars) parser.add_argument( prefix_chars[0] * 2 + 'foo', action='store_true', help='foo help') else: parser = ErrorRaisingArgumentParser( - prog='PROG', description='main description') + prog='PROG', description='main description', usage=usage) parser.add_argument( '--foo', action='store_true', help='foo help') parser.add_argument( @@ -2310,7 +2441,7 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, else: subparsers_kwargs['help'] = 'command help' subparsers = parser.add_subparsers(**subparsers_kwargs) - self.assertRaisesRegex(argparse.ArgumentError, + self.assertRaisesRegex(ValueError, 'cannot have multiple subparser arguments', parser.add_subparsers) @@ -2333,7 +2464,8 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, parser2.add_argument('z', type=complex, nargs='*', help='z help') # add third sub-parser - parser3_kwargs = dict(description='3 description') + parser3_kwargs = dict(description='3 description', + usage='PROG --foo bar 3 t ...') if subparser_help: parser3_kwargs['help'] = '3 help' parser3 = subparsers.add_parser('3', **parser3_kwargs) @@ -2355,6 +2487,47 @@ def test_parse_args_failures(self): args = args_str.split() self.assertArgumentParserError(self.parser.parse_args, args) + def test_parse_args_failures_details(self): + for args_str, usage_str, error_str in [ + ('', + 'usage: PROG [-h] [--foo] bar {1,2,3} ...', + 'PROG: error: the following arguments are required: bar'), + ('0.5 1 -y', + 'usage: PROG bar 1 [-h] [-w W] {a,b,c}', + 'PROG bar 1: error: the following arguments are required: x'), + ('0.5 3', + 'usage: PROG --foo bar 3 t ...', + 'PROG bar 3: error: the following arguments are required: t'), + ]: + with self.subTest(args_str): + args = args_str.split() + with self.assertRaises(ArgumentParserError) as cm: + self.parser.parse_args(args) + self.assertEqual(cm.exception.args[0], 'SystemExit') + self.assertEqual(cm.exception.args[2], f'{usage_str}\n{error_str}\n') + + def test_parse_args_failures_details_custom_usage(self): + parser = self._get_parser(usage='PROG [--foo] bar 1 [-w W] {a,b,c}\n' + ' PROG --foo bar 3 t ...') + for args_str, usage_str, error_str in [ + ('', + 'usage: PROG [--foo] bar 1 [-w W] {a,b,c}\n' + ' PROG --foo bar 3 t ...', + 'PROG: error: the following arguments are required: bar'), + ('0.5 1 -y', + 'usage: PROG bar 1 [-h] [-w W] {a,b,c}', + 'PROG bar 1: error: the following arguments are required: x'), + ('0.5 3', + 'usage: PROG --foo bar 3 t ...', + 'PROG bar 3: error: the following arguments are required: t'), + ]: + with self.subTest(args_str): + args = args_str.split() + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args(args) + self.assertEqual(cm.exception.args[0], 'SystemExit') + self.assertEqual(cm.exception.args[2], f'{usage_str}\n{error_str}\n') + def test_parse_args(self): # check some non-failure cases: self.assertEqual( @@ -2508,18 +2681,6 @@ def test_required_subparsers_no_destination_error(self): 'error: the following arguments are required: {foo,bar}\n$' ) - def test_wrong_argument_subparsers_no_destination_error(self): - parser = ErrorRaisingArgumentParser() - subparsers = parser.add_subparsers(required=True) - subparsers.add_parser('foo') - subparsers.add_parser('bar') - with self.assertRaises(ArgumentParserError) as excinfo: - parser.parse_args(('baz',)) - self.assertRegex( - excinfo.exception.stderr, - r"error: argument {foo,bar}: invalid choice: 'baz' \(choose from foo, bar\)\n$" - ) - def test_optional_subparsers(self): parser = ErrorRaisingArgumentParser() subparsers = parser.add_subparsers(dest='command', required=False) @@ -2655,6 +2816,29 @@ def test_parser_command_help(self): --foo foo help ''')) + def assert_bad_help(self, context_type, func, *args, **kwargs): + with self.assertRaisesRegex(ValueError, 'badly formed help string') as cm: + func(*args, **kwargs) + self.assertIsInstance(cm.exception.__context__, context_type) + + def test_invalid_subparsers_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(ValueError, parser.add_subparsers, help='%Y-%m-%d') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(KeyError, parser.add_subparsers, help='%(spam)s') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(TypeError, parser.add_subparsers, help='%(prog)d') + + def test_invalid_subparser_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + subparsers = parser.add_subparsers() + self.assert_bad_help(ValueError, subparsers.add_parser, '1', + help='%Y-%m-%d') + self.assert_bad_help(KeyError, subparsers.add_parser, '1', + help='%(spam)s') + self.assert_bad_help(TypeError, subparsers.add_parser, '1', + help='%(prog)d') + def test_subparser_title_help(self): parser = ErrorRaisingArgumentParser(prog='PROG', description='main description') @@ -2796,10 +2980,43 @@ def test_interleaved_groups(self): result = parser.parse_args('1 2 3 4'.split()) self.assertEqual(expected, result) +class TestGroupConstructor(TestCase): + def test_group_prefix_chars(self): + parser = ErrorRaisingArgumentParser() + msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + with self.assertWarns(DeprecationWarning) as cm: + parser.add_argument_group(prefix_chars='-+') + self.assertEqual(msg, str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + def test_group_prefix_chars_default(self): + # "default" isn't quite the right word here, but it's the same as + # the parser's default prefix so it's a good test + parser = ErrorRaisingArgumentParser() + msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + with self.assertWarns(DeprecationWarning) as cm: + parser.add_argument_group(prefix_chars='-') + self.assertEqual(msg, str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + def test_nested_argument_group(self): + parser = argparse.ArgumentParser() + g = parser.add_argument_group() + self.assertRaisesRegex(ValueError, + 'argument groups cannot be nested', + g.add_argument_group) + # =================== # Parent parser tests # =================== +@force_not_colorized_test_class class TestParentParsers(TestCase): """Tests that parsers can be created with parent parsers""" @@ -2832,8 +3049,6 @@ def setUp(self): group.add_argument('-a', action='store_true') group.add_argument('-b', action='store_true') - self.main_program = os.path.basename(sys.argv[0]) - def test_single_parent(self): parser = ErrorRaisingArgumentParser(parents=[self.wxyz_parent]) self.assertEqual(parser.parse_args('-y 1 2 --w 3'.split()), @@ -2844,7 +3059,7 @@ def test_single_parent_mutex(self): parser = ErrorRaisingArgumentParser(parents=[self.ab_mutex_parent]) self._test_mutex_ab(parser.parse_args) - def test_single_granparent_mutex(self): + def test_single_grandparent_mutex(self): parents = [self.ab_mutex_parent] parser = ErrorRaisingArgumentParser(add_help=False, parents=parents) parser = ErrorRaisingArgumentParser(parents=[parser]) @@ -2923,11 +3138,10 @@ def test_subparser_parents_mutex(self): def test_parent_help(self): parents = [self.abcd_parent, self.wxyz_parent] - parser = ErrorRaisingArgumentParser(parents=parents) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=parents) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-b B] [--d D] [--w W] [-y Y] a z + usage: PROG [-h] [-b B] [--d D] [--w W] [-y Y] a z positional arguments: a @@ -2943,7 +3157,7 @@ def test_parent_help(self): x: -y Y - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_groups_parents(self): parent = ErrorRaisingArgumentParser(add_help=False) @@ -2953,15 +3167,14 @@ def test_groups_parents(self): m = parent.add_mutually_exclusive_group() m.add_argument('-y') m.add_argument('-z') - parser = ErrorRaisingArgumentParser(parents=[parent]) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=[parent]) self.assertRaises(ArgumentParserError, parser.parse_args, ['-y', 'Y', '-z', 'Z']) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-w W] [-x X] [-y Y | -z Z] + usage: PROG [-h] [-w W] [-x X] [-y Y | -z Z] options: -h, --help show this help message and exit @@ -2973,7 +3186,7 @@ def test_groups_parents(self): -w W -x X - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_wrong_type_parents(self): self.assertRaises(TypeError, ErrorRaisingArgumentParser, parents=[1]) @@ -3011,6 +3224,7 @@ def test_mutex_groups_parents(self): # Mutually exclusive group tests # ============================== +@force_not_colorized_test_class class TestMutuallyExclusiveGroupErrors(TestCase): def test_invalid_add_argument_group(self): @@ -3049,6 +3263,29 @@ def test_help(self): ''' self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_optional_order(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--foo') + group.add_argument('bar', nargs='?') + expected = '''\ + usage: PROG [-h] (--foo FOO | bar) + + positional arguments: + bar + + options: + -h, --help show this help message and exit + --foo FOO + ''' + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('bar', nargs='?') + group.add_argument('--foo') + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): self.maxDiff = None parser = ErrorRaisingArgumentParser(prog='PROG') @@ -3077,6 +3314,14 @@ def test_empty_group(self): with self.assertRaises(ValueError): parser.parse_args(['-h']) + def test_nested_mutex_groups(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument("--spam") + self.assertRaisesRegex(ValueError, + 'mutually exclusive groups cannot be nested', + g.add_mutually_exclusive_group) + class MEMixin(object): def test_failures_when_not_required(self): @@ -3108,21 +3353,25 @@ def test_successes_when_required(self): actual_ns = parse_args(args_string.split()) self.assertEqual(actual_ns, expected_ns) + @force_not_colorized def test_usage_when_not_required(self): format_usage = self.get_parser(required=False).format_usage expected_usage = self.usage_when_not_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_usage_when_required(self): format_usage = self.get_parser(required=True).format_usage expected_usage = self.usage_when_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_help_when_not_required(self): format_help = self.get_parser(required=False).format_help help = self.usage_when_not_required + self.help self.assertEqual(format_help(), textwrap.dedent(help)) + @force_not_colorized def test_help_when_required(self): format_help = self.get_parser(required=True).format_help help = self.usage_when_required + self.help @@ -3444,55 +3693,6 @@ def get_parser(self, required): -c c help ''' -class TestMutuallyExclusiveNested(MEMixin, TestCase): - - # Nesting mutually exclusive groups is an undocumented feature - # that came about by accident through inheritance and has been - # the source of many bugs. It is deprecated and this test should - # eventually be removed along with it. - - def get_parser(self, required): - parser = ErrorRaisingArgumentParser(prog='PROG') - group = parser.add_mutually_exclusive_group(required=required) - group.add_argument('-a') - group.add_argument('-b') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - group2 = group.add_mutually_exclusive_group(required=required) - group2.add_argument('-c') - group2.add_argument('-d') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - group3 = group2.add_mutually_exclusive_group(required=required) - group3.add_argument('-e') - group3.add_argument('-f') - return parser - - usage_when_not_required = '''\ - usage: PROG [-h] [-a A | -b B | [-c C | -d D | [-e E | -f F]]] - ''' - usage_when_required = '''\ - usage: PROG [-h] (-a A | -b B | (-c C | -d D | (-e E | -f F))) - ''' - - help = '''\ - - options: - -h, --help show this help message and exit - -a A - -b B - -c C - -d D - -e E - -f F - ''' - - # We are only interested in testing the behavior of format_usage(). - test_failures_when_not_required = None - test_failures_when_required = None - test_successes_when_not_required = None - test_successes_when_required = None - class TestMutuallyExclusiveOptionalOptional(MEMixin, TestCase): def get_parser(self, required=None): @@ -3843,11 +4043,13 @@ def _test(self, tester, parser_text): tester.maxDiff = None tester.assertEqual(expected_text, parser_text) + @force_not_colorized def test_format(self, tester): parser = self._get_parser(tester) format = getattr(parser, 'format_%s' % self.func_suffix) self._test(tester, format()) + @force_not_colorized def test_print(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -3860,6 +4062,7 @@ def test_print(self, tester): setattr(sys, self.std_name, old_stream) self._test(tester, parser_text) + @force_not_colorized def test_print_file(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -4601,6 +4804,7 @@ class TestHelpUsageMetavarsSpacesParentheses(HelpTestCase): version = '' +@force_not_colorized_test_class class TestHelpUsageNoWhitespaceCrash(TestCase): def test_all_suppressed_mutex_followed_by_long_arg(self): @@ -4663,25 +4867,6 @@ def test_all_suppressed_mutex_with_optional_nargs(self): usage = 'usage: PROG [-h]\n' self.assertEqual(parser.format_usage(), usage) - def test_nested_mutex_groups(self): - parser = argparse.ArgumentParser(prog='PROG') - g = parser.add_mutually_exclusive_group() - g.add_argument("--spam") - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - gg = g.add_mutually_exclusive_group() - gg.add_argument("--hax") - gg.add_argument("--hox", help=argparse.SUPPRESS) - gg.add_argument("--hex") - g.add_argument("--eggs") - parser.add_argument("--num") - - usage = textwrap.dedent('''\ - usage: PROG [-h] [--spam SPAM | [--hax HAX | --hex HEX] | --eggs EGGS] - [--num NUM] - ''') - self.assertEqual(parser.format_usage(), usage) - def test_long_mutex_groups_wrap(self): parser = argparse.ArgumentParser(prog='PROG') g = parser.add_mutually_exclusive_group() @@ -5301,11 +5486,61 @@ def custom_type(string): version = '' -class TestHelpUsageLongSubparserCommand(TestCase): - """Test that subparser commands are formatted correctly in help""" +@force_not_colorized_test_class +class TestHelpCustomHelpFormatter(TestCase): maxDiff = None - def test_parent_help(self): + def test_custom_formatter_function(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog='PROG', + prefix_chars='-+', + formatter_class=custom_formatter + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [+f FOO] spam + + positional arguments: + spam spam help + + options: + -h, --help show this help message and exit + +f, ++foo FOO foo help + ''')) + + def test_custom_formatter_class(self): + class CustomFormatter(argparse.RawTextHelpFormatter): + def __init__(self, prog): + super().__init__(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog='PROG', + prefix_chars='-+', + formatter_class=CustomFormatter + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [+f FOO] spam + + positional arguments: + spam spam help + + options: + -h, --help show this help message and exit + +f, ++foo FOO foo help + ''')) + + def test_usage_long_subparser_command(self): + """Test that subparser commands are formatted correctly in help""" def custom_formatter(prog): return argparse.RawTextHelpFormatter(prog, max_help_position=50) @@ -5371,29 +5606,45 @@ def test_missing_destination(self): self.assertTypeError(action=action) def test_invalid_option_strings(self): - self.assertValueError('--') - self.assertValueError('---') + self.assertTypeError('-', errmsg='dest= is required') + self.assertTypeError('--', errmsg='dest= is required') + self.assertTypeError('---', errmsg='dest= is required') def test_invalid_prefix(self): - self.assertValueError('--foo', '+foo') + self.assertValueError('--foo', '+foo', + errmsg='must start with a character') def test_invalid_type(self): - self.assertValueError('--foo', type='int') - self.assertValueError('--foo', type=(int, float)) + self.assertTypeError('--foo', type='int', + errmsg="'int' is not callable") + self.assertTypeError('--foo', type=(int, float), + errmsg='is not callable') def test_invalid_action(self): - self.assertValueError('-x', action='foo') - self.assertValueError('foo', action='baz') - self.assertValueError('--foo', action=('store', 'append')) + self.assertValueError('-x', action='foo', + errmsg='unknown action') + self.assertValueError('foo', action='baz', + errmsg='unknown action') + self.assertValueError('--foo', action=('store', 'append'), + errmsg='unknown action') self.assertValueError('--foo', action="store-true", errmsg='unknown action') + def test_invalid_help(self): + self.assertValueError('--foo', help='%Y-%m-%d', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(spam)s', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(prog)d', + errmsg='badly formed help string') + def test_multiple_dest(self): parser = argparse.ArgumentParser() parser.add_argument(dest='foo') - with self.assertRaises(ValueError) as cm: + with self.assertRaises(TypeError) as cm: parser.add_argument('bar', dest='baz') - self.assertIn('dest supplied twice for positional argument', + self.assertIn('dest supplied twice for positional argument,' + ' did you mean metavar?', str(cm.exception)) def test_no_argument_actions(self): @@ -5405,8 +5656,11 @@ def test_no_argument_actions(self): with self.subTest(attrs=attrs): self.assertTypeError('-x', action=action, **attrs) self.assertTypeError('x', action=action, **attrs) + self.assertValueError('x', action=action, + errmsg=f"action '{action}' is not valid for positional arguments") self.assertTypeError('-x', action=action, nargs=0) - self.assertTypeError('x', action=action, nargs=0) + self.assertValueError('x', action=action, nargs=0, + errmsg='nargs for positionals must be != 0') def test_no_argument_no_const_actions(self): # options with zero arguments @@ -5426,7 +5680,7 @@ def test_more_than_one_argument_actions(self): self.assertValueError('-x', nargs=0, action=action, errmsg=f'nargs for {action_name} actions must be != 0') self.assertValueError('spam', nargs=0, action=action, - errmsg=f'nargs for {action_name} actions must be != 0') + errmsg='nargs for positionals must be != 0') # const is disallowed with non-optional arguments for nargs in [1, '*', '+']: @@ -5529,6 +5783,7 @@ def test_conflict_error(self): self.assertRaises(argparse.ArgumentError, parser.add_argument, '--spam') + @force_not_colorized def test_resolve_error(self): get_parser = argparse.ArgumentParser parser = get_parser(prog='PROG', conflict_handler='resolve') @@ -5558,20 +5813,25 @@ def test_subparser_conflict(self): parser = argparse.ArgumentParser() sp = parser.add_subparsers() sp.add_parser('fullname', aliases=['alias']) - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'fullname') - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'alias') - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'other', aliases=['fullname']) - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'other', aliases=['alias']) + self.assertRaisesRegex(ValueError, + 'conflicting subparser: fullname', + sp.add_parser, 'fullname') + self.assertRaisesRegex(ValueError, + 'conflicting subparser: alias', + sp.add_parser, 'alias') + self.assertRaisesRegex(ValueError, + 'conflicting subparser alias: fullname', + sp.add_parser, 'other', aliases=['fullname']) + self.assertRaisesRegex(ValueError, + 'conflicting subparser alias: alias', + sp.add_parser, 'other', aliases=['alias']) # ============================= # Help and Version option tests # ============================= +@force_not_colorized_test_class class TestOptionalsHelpVersionActions(TestCase): """Test the help and version actions""" @@ -5791,6 +6051,7 @@ def test_argument_error(self): class TestArgumentTypeError(TestCase): + @force_not_colorized def test_argument_type_error(self): def spam(string): @@ -6563,9 +6824,10 @@ def test_nargs_zero(self): class TestImportStar(TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON def test(self): for name in argparse.__all__: - self.assertTrue(hasattr(argparse, name)) + self.assertHasAttr(argparse, name) def test_all_exports_everything_but_modules(self): items = [ @@ -6589,6 +6851,7 @@ def setUp(self): metavar = '' self.parser.add_argument('--proxy', metavar=metavar) + @force_not_colorized def test_help_with_metavar(self): help_text = self.parser.format_help() self.assertEqual(help_text, textwrap.dedent('''\ @@ -6754,6 +7017,99 @@ def test_os_error(self): self.parser.parse_args, ['@no-such-file']) +@force_not_colorized_test_class +class TestProgName(TestCase): + source = textwrap.dedent('''\ + import argparse + parser = argparse.ArgumentParser() + parser.parse_args() + ''') + + def setUp(self): + self.dirname = 'package' + os_helper.FS_NONASCII + self.addCleanup(os_helper.rmtree, self.dirname) + os.mkdir(self.dirname) + + def make_script(self, dirname, basename, *, compiled=False): + script_name = script_helper.make_script(dirname, basename, self.source) + if not compiled: + return script_name + py_compile.compile(script_name, doraise=True) + os.remove(script_name) + pyc_file = import_helper.make_legacy_pyc(script_name) + return pyc_file + + def make_zip_script(self, script_name, name_in_zip=None): + zip_name, _ = script_helper.make_zip_script(self.dirname, 'test_zip', + script_name, name_in_zip) + return zip_name + + def check_usage(self, expected, *args, **kwargs): + res = script_helper.assert_python_ok('-Xutf8', *args, '-h', **kwargs) + self.assertEqual(os.fsdecode(res.out.splitlines()[0]), + f'usage: {expected} [-h]') + + def test_script(self, compiled=False): + basename = os_helper.TESTFN + script_name = self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(os.path.basename(script_name), script_name, '-h') + + def test_script_compiled(self): + self.test_script(compiled=True) + + def test_directory(self, compiled=False): + dirname = os.path.join(self.dirname, os_helper.TESTFN) + os.mkdir(dirname) + self.make_script(dirname, '__main__', compiled=compiled) + self.check_usage(f'{py} {dirname}', dirname) + dirname2 = os.path.join(os.curdir, dirname) + self.check_usage(f'{py} {dirname2}', dirname2) + + def test_directory_compiled(self): + self.test_directory(compiled=True) + + def test_module(self, compiled=False): + basename = 'module' + os_helper.FS_NONASCII + modulename = f'{self.dirname}.{basename}' + self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(f'{py} -m {modulename}', + '-m', modulename, PYTHONPATH=os.curdir) + + def test_module_compiled(self): + self.test_module(compiled=True) + + def test_package(self, compiled=False): + basename = 'subpackage' + os_helper.FS_NONASCII + packagename = f'{self.dirname}.{basename}' + subdirname = os.path.join(self.dirname, basename) + os.mkdir(subdirname) + self.make_script(subdirname, '__main__', compiled=compiled) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename, PYTHONPATH=os.curdir) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename + '.__main__', PYTHONPATH=os.curdir) + + def test_package_compiled(self): + self.test_package(compiled=True) + + def test_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + zip_name = self.make_zip_script(script_name) + self.check_usage(f'{py} {zip_name}', zip_name) + + def test_zipfile_compiled(self): + self.test_zipfile(compiled=True) + + def test_directory_in_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + name_in_zip = 'package/subpackage/__main__' + ('.py', '.pyc')[compiled] + zip_name = self.make_zip_script(script_name, name_in_zip) + dirname = os.path.join(zip_name, 'package', 'subpackage') + self.check_usage(f'{py} {dirname}', dirname) + + def test_directory_in_zipfile_compiled(self): + self.test_directory_in_zipfile(compiled=True) + # ================= # Translation tests # ================= @@ -6764,6 +7120,257 @@ def test_translations(self): self.assertMsgidsEqual(argparse) +# =========== +# Color tests +# =========== + + +class TestColorized(TestCase): + maxDiff = None + + def setUp(self): + super().setUp() + # Ensure color even if ran with NO_COLOR=1 + self.enterContext(swap_attr(_colorize, 'can_colorize', + lambda *args, **kwargs: True)) + self.theme = _colorize.get_theme(force_color=True).argparse + + def test_argparse_color(self): + # Arrange: create a parser with a bit of everything + parser = argparse.ArgumentParser( + color=True, + description="Colorful help", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + prefix_chars="-+", + prog="PROG", + ) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-v", "--verbose", action="store_true", help="more spam" + ) + group.add_argument( + "-q", "--quiet", action="store_true", help="less spam" + ) + parser.add_argument("x", type=int, help="the base") + parser.add_argument( + "y", type=int, help="the exponent", deprecated=True + ) + parser.add_argument( + "this_indeed_is_a_very_long_action_name", + type=int, + help="the exponent", + ) + parser.add_argument( + "-o", "--optional1", action="store_true", deprecated=True + ) + parser.add_argument("--optional2", help="pick one") + parser.add_argument("--optional3", choices=("X", "Y", "Z")) + parser.add_argument( + "--optional4", choices=("X", "Y", "Z"), help="pick one" + ) + parser.add_argument( + "--optional5", choices=("X", "Y", "Z"), help="pick one" + ) + parser.add_argument( + "--optional6", choices=("X", "Y", "Z"), help="pick one" + ) + parser.add_argument( + "-p", + "--optional7", + choices=("Aaaaa", "Bbbbb", "Ccccc", "Ddddd"), + help="pick one", + ) + + parser.add_argument("+f") + parser.add_argument("++bar") + parser.add_argument("-+baz") + parser.add_argument("-c", "--count") + + subparsers = parser.add_subparsers( + title="subcommands", + description="valid subcommands", + help="additional help", + ) + subparsers.add_parser("sub1", deprecated=True, help="sub1 help") + sub2 = subparsers.add_parser("sub2", deprecated=True, help="sub2 help") + sub2.add_argument("--baz", choices=("X", "Y", "Z"), help="baz help") + + prog = self.theme.prog + heading = self.theme.heading + long = self.theme.summary_long_option + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + # Act + help_text = parser.format_help() + + # Assert + self.assertEqual( + help_text, + textwrap.dedent( + f"""\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}-v{reset} | {short}-q{reset}] [{short}-o{reset}] [{long}--optional2 {label}OPTIONAL2{reset}] [{long}--optional3 {label}{{X,Y,Z}}{reset}] + [{long}--optional4 {label}{{X,Y,Z}}{reset}] [{long}--optional5 {label}{{X,Y,Z}}{reset}] [{long}--optional6 {label}{{X,Y,Z}}{reset}] + [{short}-p {label}{{Aaaaa,Bbbbb,Ccccc,Ddddd}}{reset}] [{short}+f {label}F{reset}] [{long}++bar {label}BAR{reset}] [{long}-+baz {label}BAZ{reset}] + [{short}-c {label}COUNT{reset}] + {pos}x{reset} {pos}y{reset} {pos}this_indeed_is_a_very_long_action_name{reset} {pos}{{sub1,sub2}} ...{reset} + + Colorful help + + {heading}positional arguments:{reset} + {pos_b}x{reset} the base + {pos_b}y{reset} the exponent + {pos_b}this_indeed_is_a_very_long_action_name{reset} + the exponent + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}-v{reset}, {long_b}--verbose{reset} more spam (default: False) + {short_b}-q{reset}, {long_b}--quiet{reset} less spam (default: False) + {short_b}-o{reset}, {long_b}--optional1{reset} + {long_b}--optional2{reset} {label_b}OPTIONAL2{reset} + pick one (default: None) + {long_b}--optional3{reset} {label_b}{{X,Y,Z}}{reset} + {long_b}--optional4{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) + {long_b}--optional5{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) + {long_b}--optional6{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) + {short_b}-p{reset}, {long_b}--optional7{reset} {label_b}{{Aaaaa,Bbbbb,Ccccc,Ddddd}}{reset} + pick one (default: None) + {short_b}+f{reset} {label_b}F{reset} + {long_b}++bar{reset} {label_b}BAR{reset} + {long_b}-+baz{reset} {label_b}BAZ{reset} + {short_b}-c{reset}, {long_b}--count{reset} {label_b}COUNT{reset} + + {heading}subcommands:{reset} + valid subcommands + + {pos_b}{{sub1,sub2}}{reset} additional help + {pos_b}sub1{reset} sub1 help + {pos_b}sub2{reset} sub2 help + """ + ), + ) + + def test_argparse_color_usage(self): + # Arrange + parser = argparse.ArgumentParser( + add_help=False, + color=True, + description="Test prog and usage colors", + prog="PROG", + usage="[prefix] %(prog)s [suffix]", + ) + heading = self.theme.heading + prog = self.theme.prog + reset = self.theme.reset + usage = self.theme.prog_extra + + # Act + help_text = parser.format_help() + + # Assert + self.assertEqual( + help_text, + textwrap.dedent( + f"""\ + {heading}usage: {reset}{usage}[prefix] {prog}PROG{reset}{usage} [suffix]{reset} + + Test prog and usage colors + """ + ), + ) + + def test_custom_formatter_function(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog="PROG", + prefix_chars="-+", + formatter_class=custom_formatter, + color=True, + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + prog = self.theme.prog + heading = self.theme.heading + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent(f'''\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} + + {heading}positional arguments:{reset} + {pos_b}spam{reset} spam help + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + ''')) + + def test_custom_formatter_class(self): + class CustomFormatter(argparse.RawTextHelpFormatter): + def __init__(self, prog): + super().__init__(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog="PROG", + prefix_chars="-+", + formatter_class=CustomFormatter, + color=True, + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + prog = self.theme.prog + heading = self.theme.heading + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent(f'''\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} + + {heading}positional arguments:{reset} + {pos_b}spam{reset} spam help + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + ''')) + + def test_subparser_prog_is_stored_without_color(self): + parser = argparse.ArgumentParser(prog='complex', color=True) + sub = parser.add_subparsers(dest='command') + demo_parser = sub.add_parser('demo') + + self.assertNotIn('\x1b[', demo_parser.prog) + + demo_parser.color = False + help_text = demo_parser.format_help() + self.assertNotIn('\x1b[', help_text) + + def tearDownModule(): # Remove global references to avoid looking like we have refleaks. RFile.seen = {} diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 38fd9ab95ba..cbba54a3bf9 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2010,21 +2010,18 @@ def test_construct_singletons(self): self.assertRaises(TypeError, tp, 1, 2) self.assertRaises(TypeError, tp, a=1, b=2) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_warning_notimplemented(self): - # Issue #35712: NotImplemented is a sentinel value that should never + def test_bool_notimplemented(self): + # GH-79893: NotImplemented is a sentinel value that should never # be evaluated in a boolean context (virtually all such use cases # are a result of accidental misuse implementing rich comparison # operations in terms of one another). - # For the time being, it will continue to evaluate as a true value, but - # issue a deprecation warning (with the eventual intent to make it - # a TypeError). - self.assertWarns(DeprecationWarning, bool, NotImplemented) - with self.assertWarns(DeprecationWarning): - self.assertTrue(NotImplemented) - with self.assertWarns(DeprecationWarning): - self.assertFalse(not NotImplemented) + msg = "NotImplemented should not be used in a boolean context" + self.assertRaisesRegex(TypeError, msg, bool, NotImplemented) + with self.assertRaisesRegex(TypeError, msg): + if NotImplemented: + pass + with self.assertRaisesRegex(TypeError, msg): + not NotImplemented class TestBreakpoint(unittest.TestCase): diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index df102fe1986..35573fd9f01 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -987,10 +987,11 @@ def assertFailure(self, *args): self.assertCLIFails(*args) self.assertCmdFails(*args) + @support.force_not_colorized def test_help(self): stdout = self.run_cmd_ok('-h') self.assertIn(b'usage:', stdout) - self.assertIn(b'calendar.py', stdout) + self.assertIn(b' -m calendar ', stdout) self.assertIn(b'--help', stdout) # special case: stdout but sys.exit() @@ -1089,6 +1090,7 @@ def test_option_months(self): output = run('--months', '1', '2004') self.assertIn(conv('\nMo Tu We Th Fr Sa Su\n'), output) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_option_type(self): self.assertFailure('-t') self.assertFailure('--type') @@ -1097,7 +1099,7 @@ def test_option_type(self): output = run('--type', 'text', '2004') self.assertEqual(output, conv(result_2004_text)) output = run('--type', 'html', '2004') - self.assertEqual(output[:6], b'Calendar for 2004', output) def test_html_output_current_year(self): diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index a4b36a90d88..6d3c91b0b6d 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -6,7 +6,6 @@ import test.support import unittest import unittest.mock -from importlib.resources.abc import Traversable from pathlib import Path import ensurepip diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 323f4ee4c61..aae5c2b1ce3 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -374,6 +374,7 @@ class F(C, A): self.assertEqual(F.__annotations__, {}) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_var_annot_metaclass_semantics(self): class CMeta(type): @classmethod @@ -403,6 +404,7 @@ def test_var_annot_in_module(self): with self.assertRaises(NameError): ann_module3.D_bad_ann(5) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_var_annot_simple_exec(self): gns = {}; lns= {} exec("'docstring'\n" diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 1c0c38ee042..e4d44e3d38c 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -248,6 +248,7 @@ def test_chain_from_iterable(self): self.assertRaises(TypeError, list, chain.from_iterable([2, 3])) self.assertEqual(list(islice(chain.from_iterable(repeat(range(5))), 2)), [0, 1]) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_chain_reducible(self): for oper in [copy.deepcopy] + picklecopiers: @@ -567,6 +568,7 @@ def test_combinatorics(self): self.assertEqual(comb, list(filter(set(perm).__contains__, cwr))) # comb: cwr that is a perm self.assertEqual(comb, sorted(set(cwr) & set(perm))) # comb: both a cwr and a perm + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_compress(self): self.assertEqual(list(compress(data='ABCDEF', selectors=[1,0,1,0,1,1])), list('ACEF')) @@ -601,6 +603,7 @@ def test_compress(self): next(testIntermediate) self.assertEqual(list(op(testIntermediate)), list(result2)) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_count(self): self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)]) @@ -1035,6 +1038,7 @@ def test_filter(self): c = filter(isEven, range(6)) self.pickletest(proto, c) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_filterfalse(self): self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5]) @@ -1142,6 +1146,7 @@ def test_zip_longest_tuple_reuse(self): ids = list(map(id, list(zip_longest('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_zip_longest_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -1365,6 +1370,7 @@ def test_product_tuple_reuse(self): self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_product_pickling(self): # check copy, deepcopy, pickle @@ -1393,6 +1399,7 @@ def test_product_issue_25021(self): p.__setstate__((0, 0, 0x1000)) # will access tuple element 1 if not clamped self.assertRaises(StopIteration, next, p) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_repeat(self): self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a']) @@ -1458,6 +1465,7 @@ def test_map(self): c = map(tupleize, 'abc', count()) self.pickletest(proto, c) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_starmap(self): self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))), @@ -1582,6 +1590,7 @@ def __index__(self): self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50), IntLike(5))), list(range(10,50,5))) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_takewhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] @@ -1941,6 +1950,7 @@ class TestExamples(unittest.TestCase): def test_accumulate(self): self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15]) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_accumulate_reducible(self): # check copy, deepcopy, pickle diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 343e5dd7a6c..b6b95143621 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -745,9 +745,6 @@ def test_init(self): def test_issue5449(self): super().test_issue5449() - def test_pickling(self): - super().test_pickling() - def test_read(self): super().test_read() @@ -907,9 +904,6 @@ def test_newline_none(self): def test_newlines_property(self): super().test_newlines_property() - def test_pickling(self): - super().test_pickling() - def test_read(self): super().test_read() @@ -998,17 +992,11 @@ def test_newline_cr(self): def test_newline_crlf(self): super().test_newline_crlf() - def test_newline_default(self): - super().test_newline_default() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_newline_empty(self): super().test_newline_empty() - def test_newline_lf(self): - super().test_newline_lf() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_newline_none(self): diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index b599c6d8c8d..59c74fd0d41 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -293,8 +293,6 @@ class M(ModuleType): melon = Descr() self.assertRaises(RuntimeError, getattr, M("mymod"), "melon") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazy_create_annotations(self): # module objects lazy create their __annotations__ dict on demand. # the annotations dict is stored in module.__dict__. @@ -334,7 +332,11 @@ def test_annotations_getset_raises(self): del foo.__annotations__ def test_annotations_are_created_correctly(self): - ann_module4 = import_helper.import_fresh_module('test.typinganndata.ann_module4') + ann_module4 = import_helper.import_fresh_module( + 'test.typinganndata.ann_module4', + ) + self.assertFalse("__annotations__" in ann_module4.__dict__) + self.assertEqual(ann_module4.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in ann_module4.__dict__) del ann_module4.__annotations__ self.assertFalse("__annotations__" in ann_module4.__dict__) diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index 72488b2bb6b..0e1a4fc50d2 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -36,11 +36,15 @@ def test_default_annotations_exist(self): class C: pass self.assertEqual(C.__annotations__, {}) + # TODO: RustPython - test expectation changed in 3.14 due to PEP 649 + @unittest.expectedFailure def test_use_existing_annotations(self): ns = {'__annotations__': {1: 2}} exec('x: int', ns) self.assertEqual(ns['__annotations__'], {'x': int, 1: 2}) + # TODO: RustPython - test expectation changed in 3.14 due to PEP 649 + @unittest.expectedFailure def test_do_not_recreate_annotations(self): # Don't rely on the existence of the '__annotations__' global. with support.swap_item(globals(), '__annotations__', {}): diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 7271696a191..0b634ea5dd0 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -131,8 +131,6 @@ def loads(self, buf, **kwds): def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_c_methods() - - # TODO: RUSTPYTHON @unittest.expectedFailure def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes @@ -227,7 +225,6 @@ def loads(self, buf, **kwds): def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_c_methods() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_badly_escaped_string(self): # TODO(RUSTPYTHON): Remove this test when it passes @@ -243,7 +240,6 @@ def test_correctly_quoted_string(self): # TODO(RUSTPYTHON): Remove this test whe def test_load_python2_str_as_bytes(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_load_python2_str_as_bytes() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index 42a20da00b1..f4ce1c75b87 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -97,8 +97,6 @@ def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_oob_buffers_writable_to_readonly() - - # TODO: RUSTPYTHON @unittest.expectedFailure def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index df279bd9652..56ed457882c 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -8,6 +8,7 @@ import test.support from test import support from test.support.script_helper import assert_python_ok +from test.support import import_helper from test.support import os_helper from test.support import socket_helper from test.support import captured_stderr @@ -308,8 +309,7 @@ def test_getuserbase(self): with EnvironmentVarGuard() as environ: environ['PYTHONUSERBASE'] = 'xoxo' - self.assertTrue(site.getuserbase().startswith('xoxo'), - site.getuserbase()) + self.assertTrue(site.getuserbase().startswith('xoxo')) @unittest.skipUnless(HAS_USER_SITE, 'need user site') def test_getusersitepackages(self): @@ -319,7 +319,7 @@ def test_getusersitepackages(self): # the call sets USER_BASE *and* USER_SITE self.assertEqual(site.USER_SITE, user_site) - self.assertTrue(user_site.startswith(site.USER_BASE), user_site) + self.assertTrue(user_site.startswith(site.USER_BASE)) self.assertEqual(site.USER_BASE, site.getuserbase()) def test_getsitepackages(self): @@ -362,11 +362,10 @@ def test_no_home_directory(self): environ.unset('PYTHONUSERBASE', 'APPDATA') user_base = site.getuserbase() - self.assertTrue(user_base.startswith('~' + os.sep), - user_base) + self.assertTrue(user_base.startswith('~' + os.sep)) user_site = site.getusersitepackages() - self.assertTrue(user_site.startswith(user_base), user_site) + self.assertTrue(user_site.startswith(user_base)) with mock.patch('os.path.isdir', return_value=False) as mock_isdir, \ mock.patch.object(site, 'addsitedir') as mock_addsitedir, \ @@ -515,7 +514,7 @@ def test_sitecustomize_executed(self): # If sitecustomize is available, it should have been imported. if "sitecustomize" not in sys.modules: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError: pass else: @@ -578,6 +577,17 @@ def test_license_exists_at_url(self): code = e.code self.assertEqual(code, 200, msg="Can't find " + url) + @support.cpython_only + def test_lazy_imports(self): + import_helper.ensure_lazy_imports("site", [ + "io", + "locale", + "traceback", + "atexit", + "warnings", + "textwrap", + ]) + class StartupImportTests(unittest.TestCase): @@ -843,12 +853,15 @@ def get_excepted_output(self, *args): return 10, None def invoke_command_line(self, *args): - args = ["-m", "site", *args] + cmd_args = [] + if sys.flags.no_user_site: + cmd_args.append("-s") + cmd_args.extend(["-m", "site", *args]) with EnvironmentVarGuard() as env: env["PYTHONUTF8"] = "1" env["PYTHONIOENCODING"] = "utf-8" - proc = spawn_python(*args, text=True, env=env, + proc = spawn_python(*cmd_args, text=True, env=env, encoding='utf-8', errors='replace') output = kill_python(proc) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 17cbaa92218..d96a3a6a5d4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3803,6 +3803,7 @@ def __init__(self): self.assertNotIsInstance(B(), P) self.assertNotIsInstance(C(), P) + @unittest.expectedFailure # TODO: RUSTPYTHON; test doesn't include PEP 649 attrs def test_non_protocol_subclasses(self): class P(Protocol): x = 1 @@ -6868,6 +6869,7 @@ def test_get_type_hints_modules_forwardref(self): 'default_b': Optional[mod_generics_cache.B]} self.assertEqual(gth(mod_generics_cache), mgc_hints) + @unittest.expectedFailure # TODO: RUSTPYTHON; test expects outdated result def test_get_type_hints_classes(self): self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) diff --git a/Lib/test/typinganndata/ann_module.py b/Lib/test/typinganndata/ann_module.py index 5081e6b5834..e1a1792cb4a 100644 --- a/Lib/test/typinganndata/ann_module.py +++ b/Lib/test/typinganndata/ann_module.py @@ -8,8 +8,6 @@ from typing import Optional from functools import wraps -__annotations__[1] = 2 - class C: x = 5; y: Optional['C'] = None @@ -18,8 +16,6 @@ class C: x: int = 5; y: str = x; f: Tuple[int, int] class M(type): - - __annotations__['123'] = 123 o: type = object (pars): bool = True diff --git a/Lib/typing.py b/Lib/typing.py index a7397356d65..77caac9eed1 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -161,7 +161,17 @@ ] -def _type_convert(arg, module=None, *, allow_special_forms=False): +class _LazyAnnotationLib: + def __getattr__(self, attr): + global _lazy_annotationlib + import annotationlib + _lazy_annotationlib = annotationlib + return getattr(annotationlib, attr) + +_lazy_annotationlib = _LazyAnnotationLib() + + +def _type_convert(arg, module=None, *, allow_special_forms=False, owner=None): """For converting None to type(None), and strings to ForwardRef.""" if arg is None: return type(None) @@ -170,7 +180,7 @@ def _type_convert(arg, module=None, *, allow_special_forms=False): return arg -def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False): +def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False, owner=None): """Check that the argument is a type, and return it (internal helper). As a special case, accept None and return type(None) instead. Also wrap strings @@ -188,7 +198,7 @@ def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms= if is_argument: invalid_generic_forms += (Final,) - arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms) + arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms, owner=owner) if (isinstance(arg, _GenericAlias) and arg.__origin__ in invalid_generic_forms): raise TypeError(f"{arg} is not valid as type argument") @@ -1918,6 +1928,7 @@ class _TypingEllipsis: '__init__', '__module__', '__new__', '__slots__', '__subclasshook__', '__weakref__', '__class_getitem__', '__match_args__', '__static_attributes__', '__firstlineno__', + '__annotate__', '__annotate_func__', '__annotations_cache__', }) # These special attributes will be not collected as protocol members. @@ -2442,7 +2453,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {}) else: base_globals = globalns - ann = base.__dict__.get('__annotations__', {}) + ann = _lazy_annotationlib.get_annotations(base) if isinstance(ann, types.GetSetDescriptorType): ann = {} base_locals = dict(vars(base)) if localns is None else localns @@ -2476,7 +2487,10 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): localns = globalns elif localns is None: localns = globalns - hints = getattr(obj, '__annotations__', None) + try: + hints = _lazy_annotationlib.get_annotations(obj) + except TypeError: + hints = getattr(obj, '__annotations__', None) if hints is None: # Return empty annotations for something that _could_ have them. if isinstance(obj, _allowed_types): @@ -2992,7 +3006,8 @@ def _make_nmtuple(name, types, module, defaults = ()): '_fields', '_field_defaults', '_make', '_replace', '_asdict', '_source'}) -_special = frozenset({'__module__', '__name__', '__annotations__'}) +_special = frozenset({'__module__', '__name__', '__annotations__', '__annotate__', + '__annotate_func__', '__annotations_cache__'}) class NamedTupleMeta(type): @@ -3003,7 +3018,13 @@ def __new__(cls, typename, bases, ns): raise TypeError( 'can only inherit from a NamedTuple type and Generic') bases = tuple(tuple if base is _NamedTuple else base for base in bases) - types = ns.get('__annotations__', {}) + if "__annotations__" in ns: + types = ns["__annotations__"] + elif (annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: + types = _lazy_annotationlib.call_annotate_function( + annotate, _lazy_annotationlib.Format.VALUE) + else: + types = {} default_names = [] for field_name in types: if field_name in ns: @@ -3158,16 +3179,26 @@ def __new__(cls, name, bases, ns, total=True): else: generic_base = () + ns_annotations = ns.pop('__annotations__', None) + tp_dict = type.__new__(_TypedDictMeta, name, (*generic_base, dict), ns) if not hasattr(tp_dict, '__orig_bases__'): tp_dict.__orig_bases__ = bases - annotations = {} - own_annotations = ns.get('__annotations__', {}) + if ns_annotations is not None: + own_annotate = None + own_annotations = ns_annotations + elif (own_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: + own_annotations = _lazy_annotationlib.call_annotate_function( + own_annotate, _lazy_annotationlib.Format.FORWARDREF, owner=tp_dict + ) + else: + own_annotate = None + own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" - own_annotations = { - n: _type_check(tp, msg, module=tp_dict.__module__) + own_checked_annotations = { + n: _type_check(tp, msg, owner=tp_dict, module=tp_dict.__module__) for n, tp in own_annotations.items() } required_keys = set() @@ -3176,8 +3207,6 @@ def __new__(cls, name, bases, ns, total=True): mutable_keys = set() for base in bases: - annotations.update(base.__dict__.get('__annotations__', {})) - base_required = base.__dict__.get('__required_keys__', set()) required_keys |= base_required optional_keys -= base_required @@ -3189,8 +3218,7 @@ def __new__(cls, name, bases, ns, total=True): readonly_keys.update(base.__dict__.get('__readonly_keys__', ())) mutable_keys.update(base.__dict__.get('__mutable_keys__', ())) - annotations.update(own_annotations) - for annotation_key, annotation_type in own_annotations.items(): + for annotation_key, annotation_type in own_checked_annotations.items(): qualifiers = set(_get_typeddict_qualifiers(annotation_type)) if Required in qualifiers: is_required = True @@ -3221,7 +3249,36 @@ def __new__(cls, name, bases, ns, total=True): f"Required keys overlap with optional keys in {name}:" f" {required_keys=}, {optional_keys=}" ) - tp_dict.__annotations__ = annotations + + def __annotate__(format): + annos = {} + for base in bases: + if base is Generic: + continue + base_annotate = base.__annotate__ + if base_annotate is None: + continue + base_annos = _lazy_annotationlib.call_annotate_function( + base_annotate, format, owner=base) + annos.update(base_annos) + if own_annotate is not None: + own = _lazy_annotationlib.call_annotate_function( + own_annotate, format, owner=tp_dict) + if format != _lazy_annotationlib.Format.STRING: + own = { + n: _type_check(tp, msg, module=tp_dict.__module__) + for n, tp in own.items() + } + elif format == _lazy_annotationlib.Format.STRING: + own = _lazy_annotationlib.annotations_to_string(own_annotations) + elif format in (_lazy_annotationlib.Format.FORWARDREF, _lazy_annotationlib.Format.VALUE): + own = own_checked_annotations + else: + raise NotImplementedError(format) + annos.update(own) + return annos + + tp_dict.__annotate__ = __annotate__ tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) tp_dict.__readonly_keys__ = frozenset(readonly_keys) diff --git a/Lib/venv/scripts/nt/venvlauncher.exe b/Lib/venv/scripts/nt/venvlauncher.exe index 2439c22aa93..c6863b56e57 100644 Binary files a/Lib/venv/scripts/nt/venvlauncher.exe and b/Lib/venv/scripts/nt/venvlauncher.exe differ diff --git a/Lib/venv/scripts/nt/venvlaunchert.exe b/Lib/venv/scripts/nt/venvlaunchert.exe index 99f5f5e9fca..c12a7a869f4 100644 Binary files a/Lib/venv/scripts/nt/venvlaunchert.exe and b/Lib/venv/scripts/nt/venvlaunchert.exe differ diff --git a/Lib/venv/scripts/nt/venvwlauncher.exe b/Lib/venv/scripts/nt/venvwlauncher.exe index 6c43c2e9d93..d0d3733266f 100644 Binary files a/Lib/venv/scripts/nt/venvwlauncher.exe and b/Lib/venv/scripts/nt/venvwlauncher.exe differ diff --git a/Lib/venv/scripts/nt/venvwlaunchert.exe b/Lib/venv/scripts/nt/venvwlaunchert.exe index 74f40deb046..9456a9e9b4a 100644 Binary files a/Lib/venv/scripts/nt/venvwlaunchert.exe and b/Lib/venv/scripts/nt/venvwlaunchert.exe differ diff --git a/README.md b/README.md index b3ddbe4e7cf..c4f7bfb1d60 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # [RustPython](https://rustpython.github.io/) -A Python-3 (CPython >= 3.13.0) Interpreter written in Rust :snake: :scream: +A Python-3 (CPython >= 3.14.0) Interpreter written in Rust :snake: :scream: :metal:. [![Build Status](https://github.com/RustPython/RustPython/workflows/CI/badge.svg)](https://github.com/RustPython/RustPython/actions?query=workflow%3ACI) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 7782e6ec485..d479c0d0e69 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -29,7 +29,7 @@ use ruff_python_ast::{ InterpolatedStringElements, Keyword, MatchCase, ModExpression, ModModule, Operator, Parameters, Pattern, PatternMatchAs, PatternMatchClass, PatternMatchMapping, PatternMatchOr, PatternMatchSequence, PatternMatchSingleton, PatternMatchStar, PatternMatchValue, Singleton, - Stmt, StmtExpr, TString, TypeParam, TypeParamParamSpec, TypeParamTypeVar, + Stmt, StmtAnnAssign, StmtExpr, TString, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, UnaryOp, WithItem, visitor::{Visitor, walk_expr}, }; @@ -130,6 +130,10 @@ struct Compiler { ctx: CompileContext, opts: CompileOpts, in_annotation: bool, + // PEP 649: Track if we're inside a conditional block (if/for/while/etc.) + in_conditional_block: bool, + // PEP 649: Next index for conditional annotation tracking + next_conditional_annotation_index: u32, } enum DoneWithFuture { @@ -437,6 +441,8 @@ impl Compiler { }, opts, in_annotation: false, + in_conditional_block: false, + next_conditional_annotation_index: 0, } } @@ -676,6 +682,62 @@ impl Compiler { Ok(self.current_symbol_table()) } + /// Push the annotation symbol table from the next sub_table's annotation_block + /// The annotation_block is stored in the function's scope, which is the next sub_table + /// Returns true if annotation_block exists, false otherwise + fn push_annotation_symbol_table(&mut self) -> bool { + let current_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table"); + + // The annotation_block is in the next sub_table (function scope) + let next_idx = current_table.next_sub_table; + if next_idx >= current_table.sub_tables.len() { + return false; + } + + let next_table = &mut current_table.sub_tables[next_idx]; + if let Some(annotation_block) = next_table.annotation_block.take() { + self.symbol_table_stack.push(*annotation_block); + true + } else { + false + } + } + + /// Push the annotation symbol table for module/class level annotations + /// This takes annotation_block from the current symbol table (not sub_tables) + fn push_current_annotation_symbol_table(&mut self) -> bool { + let current_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table"); + + // For modules/classes, annotation_block is directly in the current table + if let Some(annotation_block) = current_table.annotation_block.take() { + self.symbol_table_stack.push(*annotation_block); + true + } else { + false + } + } + + /// Pop the annotation symbol table and restore it to the function scope's annotation_block + fn pop_annotation_symbol_table(&mut self) { + let annotation_table = self.symbol_table_stack.pop().expect("compiler bug"); + let current_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table"); + + // Restore to the next sub_table (function scope) where it came from + let next_idx = current_table.next_sub_table; + if next_idx < current_table.sub_tables.len() { + current_table.sub_tables[next_idx].annotation_block = Some(Box::new(annotation_table)); + } + } + /// Pop the current symbol table off the stack fn pop_symbol_table(&mut self) -> SymbolTable { self.symbol_table_stack.pop().expect("compiler bug") @@ -896,6 +958,12 @@ impl Compiler { cellvar_cache.insert("__classdict__".to_string()); } + // Handle implicit __conditional_annotations__ cell if needed + // Only for class scope - module scope uses NAME operations, not DEREF + if ste.has_conditional_annotations && scope_type == CompilerScope::Class { + cellvar_cache.insert("__conditional_annotations__".to_string()); + } + // Build freevars using dictbytype (FREE scope, offset by cellvars size) let mut freevar_cache = IndexSet::default(); let mut free_names: Vec<_> = ste @@ -933,6 +1001,12 @@ impl Compiler { 0, 0, ), + CompilerScope::Annotation => ( + bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED, + 0, + 1, // annotation scope takes one argument (format) + 0, + ), }; // Get private name from parent scope @@ -1060,6 +1134,81 @@ impl Compiler { unwrap_internal(self, stack_top.finalize_code(&self.opts)) } + /// Exit annotation scope - similar to exit_scope but restores annotation_block to parent + fn exit_annotation_scope(&mut self) -> CodeObject { + self.pop_annotation_symbol_table(); + + let pop = self.code_stack.pop(); + let stack_top = compiler_unwrap_option(self, pop); + unwrap_internal(self, stack_top.finalize_code(&self.opts)) + } + + /// Enter annotation scope using the symbol table's annotation_block + /// Returns false if no annotation_block exists + fn enter_annotation_scope(&mut self, func_name: &str) -> CompileResult { + if !self.push_annotation_symbol_table() { + return Ok(false); + } + + let key = self.symbol_table_stack.len() - 1; + let lineno = self.get_source_line_number().get(); + let annotate_name = format!(""); + + self.enter_scope( + &annotate_name, + CompilerScope::Annotation, + key, + lineno.to_u32(), + )?; + + // Override arg_count since enter_scope sets it to 1 but we need the varnames + // setup to be correct too + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + + // Emit format validation: if format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError + // VALUE_WITH_FAKE_GLOBALS = 2 (from annotationlib.Format) + self.emit_format_validation()?; + + Ok(true) + } + + /// Emit format parameter validation for annotation scope + /// if format > VALUE_WITH_FAKE_GLOBALS (2): raise NotImplementedError + fn emit_format_validation(&mut self) -> CompileResult<()> { + use bytecode::ComparisonOperator::Greater; + + // Load format parameter (first local variable, index 0) + emit!(self, Instruction::LoadFast(0)); + + // Load VALUE_WITH_FAKE_GLOBALS constant (2) + self.emit_load_const(ConstantData::Integer { value: 2.into() }); + + // Compare: format > 2 + emit!(self, Instruction::CompareOp { op: Greater }); + + // Jump to body if format <= 2 (comparison is false) + let body_block = self.new_block(); + emit!(self, Instruction::PopJumpIfFalse { target: body_block }); + + // Raise NotImplementedError + let not_implemented_error = self.name("NotImplementedError"); + emit!(self, Instruction::LoadGlobal(not_implemented_error)); + emit!( + self, + Instruction::RaiseVarargs { + kind: bytecode::RaiseKind::Raise + } + ); + + // Body label - continue with annotation evaluation + self.switch_to_block(body_block); + + Ok(()) + } + /// Push a new fblock // = compiler_push_fblock fn push_fblock( @@ -1495,6 +1644,8 @@ impl Compiler { symbol_table: SymbolTable, ) -> CompileResult<()> { let size_before = self.code_stack.len(); + // Set future_annotations from symbol table (detected during symbol table scan) + self.future_annotations = symbol_table.future_annotations; self.symbol_table_stack.push(symbol_table); let (doc, statements) = split_doc(&body.body, &self.opts); @@ -1506,10 +1657,24 @@ impl Compiler { emit!(self, Instruction::StoreGlobal(doc)) } + // Handle annotations based on future_annotations flag if Self::find_ann(statements) { - emit!(self, Instruction::SetupAnnotations); + if self.future_annotations { + // PEP 563: Initialize __annotations__ dict + emit!(self, Instruction::SetupAnnotations); + } else { + // PEP 649: Generate __annotate__ function FIRST (before statements) + self.compile_module_annotate(statements)?; + + // PEP 649: Initialize __conditional_annotations__ set after __annotate__ + if self.current_symbol_table().has_conditional_annotations { + emit!(self, Instruction::BuildSet { size: 0 }); + self.store_name("__conditional_annotations__")?; + } + } } + // Compile all statements self.compile_statements(statements)?; assert_eq!(self.code_stack.len(), size_before); @@ -1524,10 +1689,25 @@ impl Compiler { body: &[Stmt], symbol_table: SymbolTable, ) -> CompileResult<()> { + // Set future_annotations from symbol table (detected during symbol table scan) + self.future_annotations = symbol_table.future_annotations; self.symbol_table_stack.push(symbol_table); + // Handle annotations based on future_annotations flag if Self::find_ann(body) { - emit!(self, Instruction::SetupAnnotations); + if self.future_annotations { + // PEP 563: Initialize __annotations__ dict + emit!(self, Instruction::SetupAnnotations); + } else { + // PEP 649: Generate __annotate__ function FIRST (before statements) + self.compile_module_annotate(body)?; + + // PEP 649: Initialize __conditional_annotations__ set after __annotate__ + if self.current_symbol_table().has_conditional_annotations { + emit!(self, Instruction::BuildSet { size: 0 }); + self.store_name("__conditional_annotations__")?; + } + } } if let Some((last, body)) = body.split_last() { @@ -1650,6 +1830,7 @@ impl Compiler { Global, Deref, Name, + DictOrGlobals, // PEP 649: can_see_class_scope } let name = self.mangle(name); @@ -1666,16 +1847,18 @@ impl Compiler { // Determine the operation type based on symbol scope let is_function_like = self.ctx.in_func(); - // Look up the symbol, handling TypeParams scope specially - let (symbol_scope, _is_typeparams) = { + // Look up the symbol, handling TypeParams and Annotation scopes specially + let (symbol_scope, can_see_class_scope) = { let current_table = self.current_symbol_table(); let is_typeparams = current_table.typ == CompilerScope::TypeParams; + let is_annotation = current_table.typ == CompilerScope::Annotation; + let can_see_class = current_table.can_see_class_scope; // First try to find in current table let symbol = current_table.lookup(name.as_ref()); - // If not found and we're in TypeParams scope, try parent scope - let symbol = if symbol.is_none() && is_typeparams { + // If not found and we're in TypeParams or Annotation scope, try parent scope + let symbol = if symbol.is_none() && (is_typeparams || is_annotation) { self.symbol_table_stack .get(self.symbol_table_stack.len() - 2) // Try to get parent index .expect("Symbol has no parent! This is a compiler bug.") @@ -1684,14 +1867,46 @@ impl Compiler { symbol }; - (symbol.map(|s| s.scope), is_typeparams) + (symbol.map(|s| s.scope), can_see_class) }; - let actual_scope = symbol_scope.ok_or_else(|| { - self.error(CodegenErrorType::SyntaxError(format!( - "The symbol '{name}' must be present in the symbol table" - ))) - })?; + // Special handling for class scope implicit cell variables + // These are treated as Cell even if not explicitly marked in symbol table + // Only for LOAD operations - explicit stores like `__class__ = property(...)` + // should use STORE_NAME to store in class namespace dict + let symbol_scope = { + let current_table = self.current_symbol_table(); + if current_table.typ == CompilerScope::Class + && usage == NameUsage::Load + && (name == "__class__" + || name == "__classdict__" + || name == "__conditional_annotations__") + { + Some(SymbolScope::Cell) + } else { + symbol_scope + } + }; + + // In annotation or type params scope, missing symbols are treated as global implicit + // This allows referencing global names like Union, Optional, etc. that are imported + // at module level but not explicitly bound in the function scope + let actual_scope = match symbol_scope { + Some(scope) => scope, + None => { + let current_table = self.current_symbol_table(); + if matches!( + current_table.typ, + CompilerScope::Annotation | CompilerScope::TypeParams + ) { + SymbolScope::GlobalImplicit + } else { + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "the symbol '{name}' must be present in the symbol table" + )))); + } + } + }; // Determine operation type based on scope let op_type = match actual_scope { @@ -1705,7 +1920,11 @@ impl Compiler { } } SymbolScope::GlobalImplicit => { - if is_function_like { + // PEP 649: In annotation scope with class visibility, use DictOrGlobals + // to check classdict first before globals + if can_see_class_scope { + NameOp::DictOrGlobals + } else if is_function_like { NameOp::Global } else { NameOp::Name @@ -1765,6 +1984,25 @@ impl Compiler { }; self.emit_arg(idx, op); } + NameOp::DictOrGlobals => { + // PEP 649: First check classdict (from __classdict__ freevar), then globals + let idx = self.get_global_name_index(&name); + match usage { + NameUsage::Load => { + // Load __classdict__ first (it's a free variable in annotation scope) + let classdict_idx = self.get_free_var_index("__classdict__")?; + self.emit_arg(classdict_idx, Instruction::LoadDeref); + self.emit_arg(idx, Instruction::LoadFromDictOrGlobals); + } + // Store/Delete in annotation scope should use Name ops + NameUsage::Store => { + self.emit_arg(idx, Instruction::StoreName); + } + NameUsage::Delete => { + self.emit_arg(idx, Instruction::DeleteName); + } + } + } } Ok(()) @@ -2117,8 +2355,9 @@ impl Compiler { target, annotation, value, + simple, .. - }) => self.compile_annotated_assign(target, annotation, value.as_deref())?, + }) => self.compile_annotated_assign(target, annotation, value.as_deref(), *simple)?, Stmt::Delete(StmtDelete { targets, .. }) => { for target in targets { self.compile_delete(target)?; @@ -3306,12 +3545,19 @@ impl Compiler { // Set qualname self.set_qualname(); - // Handle docstring + // Handle docstring - store in co_consts[0] if present let (doc_str, body) = split_doc(body, &self.opts); - self.current_code_info() - .metadata - .consts - .insert_full(ConstantData::None); + if let Some(doc) = &doc_str { + // Docstring present: store in co_consts[0] and set HAS_DOCSTRING flag + self.current_code_info() + .metadata + .consts + .insert_full(ConstantData::Str { + value: doc.to_string().into(), + }); + self.current_code_info().flags |= bytecode::CodeFlags::HAS_DOCSTRING; + } + // If no docstring, don't add None to co_consts // Compile body statements self.compile_statements(body)?; @@ -3331,30 +3577,41 @@ impl Compiler { // Create function object with closure self.make_closure(code, funcflags)?; - // Handle docstring if present - if let Some(doc) = doc_str { - emit!(self, Instruction::Copy { index: 1_u32 }); - self.emit_load_const(ConstantData::Str { - value: doc.to_string().into(), - }); - emit!(self, Instruction::Swap { index: 2 }); - let doc_attr = self.name("__doc__"); - emit!(self, Instruction::StoreAttr { idx: doc_attr }); - } + // Note: docstring is now retrieved from co_consts[0] by the VM + // when HAS_DOCSTRING flag is set, so no runtime __doc__ assignment needed Ok(()) } - /// Compile function annotations - // = compiler_visit_annotations - fn visit_annotations( + /// Compile function annotations as a closure (PEP 649) + /// Returns true if an __annotate__ closure was created + /// Uses symbol table's annotation_block for proper scoping. + fn compile_annotations_closure( &mut self, + func_name: &str, parameters: &Parameters, returns: Option<&Expr>, - ) -> CompileResult { - let mut num_annotations = 0; + ) -> CompileResult { + // Try to enter annotation scope - returns false if no annotation_block exists + if !self.enter_annotation_scope(func_name)? { + return Ok(false); + } - // Handle parameter annotations + // Count annotations + let parameters_iter = core::iter::empty() + .chain(¶meters.posonlyargs) + .chain(¶meters.args) + .chain(¶meters.kwonlyargs) + .map(|x| &x.parameter) + .chain(parameters.vararg.as_deref()) + .chain(parameters.kwarg.as_deref()); + + let num_annotations: u32 = + u32::try_from(parameters_iter.filter(|p| p.annotation.is_some()).count()) + .expect("too many annotations") + + if returns.is_some() { 1 } else { 0 }; + + // Compile annotations inside the annotation scope let parameters_iter = core::iter::empty() .chain(¶meters.posonlyargs) .chain(¶meters.args) @@ -3369,20 +3626,193 @@ impl Compiler { value: self.mangle(param.name.as_str()).into_owned().into(), }); self.compile_annotation(annotation)?; - num_annotations += 1; } } - // Handle return annotation last + // Handle return annotation if let Some(annotation) = returns { self.emit_load_const(ConstantData::Str { value: "return".into(), }); self.compile_annotation(annotation)?; - num_annotations += 1; } - Ok(num_annotations) + // Build the map and return it + emit!( + self, + Instruction::BuildMap { + size: num_annotations, + } + ); + emit!(self, Instruction::ReturnValue); + + // Exit the annotation scope and get the code object + let annotate_code = self.exit_annotation_scope(); + + // Make a closure from the code object + self.make_closure(annotate_code, bytecode::MakeFunctionFlags::empty())?; + + Ok(true) + } + + /// Collect simple (non-conditional) annotations from module body + /// Returns list of (name, annotation_expr) pairs + fn collect_simple_annotations(body: &[Stmt]) -> Vec<(&str, &Expr)> { + let mut annotations = Vec::new(); + for stmt in body { + if let Stmt::AnnAssign(StmtAnnAssign { + target, + annotation, + simple, + .. + }) = stmt + && *simple + && let Expr::Name(ExprName { id, .. }) = target.as_ref() + { + annotations.push((id.as_str(), annotation.as_ref())); + } + } + annotations + } + + /// Compile module-level __annotate__ function (PEP 649) + /// Returns true if __annotate__ was created and stored + fn compile_module_annotate(&mut self, body: &[Stmt]) -> CompileResult { + // Collect simple annotations from module body first + let annotations = Self::collect_simple_annotations(body); + + if annotations.is_empty() { + return Ok(false); + } + + // Check if we have conditional annotations + let has_conditional = self.current_symbol_table().has_conditional_annotations; + + // Get parent scope type and name BEFORE pushing annotation symbol table + let parent_scope_type = self.current_symbol_table().typ; + let parent_name = self + .symbol_table_stack + .last() + .map(|t| t.name.as_str()) + .unwrap_or("module") + .to_owned(); + let scope_name = format!(""); + + // Try to push annotation symbol table from current scope + if !self.push_current_annotation_symbol_table() { + return Ok(false); + } + + // Enter annotation scope for code generation + let key = self.symbol_table_stack.len() - 1; + let lineno = self.get_source_line_number().get(); + self.enter_scope(&scope_name, CompilerScope::Annotation, key, lineno.to_u32())?; + + // Add 'format' parameter to varnames + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + + // Emit format validation: if format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError + self.emit_format_validation()?; + + if has_conditional { + // PEP 649: Build dict incrementally, checking conditional annotations + // Start with empty dict + emit!(self, Instruction::BuildMap { size: 0 }); + + // Process each annotation + for (idx, (name, annotation)) in annotations.iter().enumerate() { + // Check if index is in __conditional_annotations__ + let not_set_block = self.new_block(); + + // LOAD_CONST index + self.emit_load_const(ConstantData::Integer { value: idx.into() }); + // Load __conditional_annotations__ from appropriate scope + // Class scope: LoadDeref (freevars), Module scope: LoadGlobal + if parent_scope_type == CompilerScope::Class { + let idx = self.get_free_var_index("__conditional_annotations__")?; + emit!(self, Instruction::LoadDeref(idx)); + } else { + let cond_annotations_name = self.name("__conditional_annotations__"); + emit!(self, Instruction::LoadGlobal(cond_annotations_name)); + } + // CONTAINS_OP (in) + emit!(self, Instruction::ContainsOp(bytecode::Invert::No)); + // POP_JUMP_IF_FALSE not_set + emit!( + self, + Instruction::PopJumpIfFalse { + target: not_set_block + } + ); + + // Annotation value + self.compile_annotation(annotation)?; + // COPY dict to TOS + emit!(self, Instruction::Copy { index: 2 }); + // LOAD_CONST name + self.emit_load_const(ConstantData::Str { + value: self.mangle(name).into_owned().into(), + }); + // STORE_SUBSCR - dict[name] = value + emit!(self, Instruction::StoreSubscr); + + // not_set label + self.switch_to_block(not_set_block); + } + + // Return the dict + emit!(self, Instruction::ReturnValue); + } else { + // No conditional annotations - use simple BuildMap + let num_annotations = u32::try_from(annotations.len()).expect("too many annotations"); + + // Compile annotations inside the annotation scope + for (name, annotation) in annotations { + self.emit_load_const(ConstantData::Str { + value: self.mangle(name).into_owned().into(), + }); + self.compile_annotation(annotation)?; + } + + // Build the map and return it + emit!( + self, + Instruction::BuildMap { + size: num_annotations, + } + ); + emit!(self, Instruction::ReturnValue); + } + + // Exit annotation scope - pop symbol table, restore to parent's annotation_block, and get code + let annotation_table = self.pop_symbol_table(); + // Restore annotation_block to module's symbol table + self.symbol_table_stack + .last_mut() + .expect("no module symbol table") + .annotation_block = Some(Box::new(annotation_table)); + // Exit code scope + let pop = self.code_stack.pop(); + let annotate_code = unwrap_internal( + self, + compiler_unwrap_option(self, pop).finalize_code(&self.opts), + ); + + // Make a closure from the code object + self.make_closure(annotate_code, bytecode::MakeFunctionFlags::empty())?; + + // Store as __annotate_func__ for classes, __annotate__ for modules + let name = if parent_scope_type == CompilerScope::Class { + "__annotate_func__" + } else { + "__annotate__" + }; + self.store_name(name)?; + + Ok(true) } // = compiler_function @@ -3449,18 +3879,12 @@ impl Compiler { } } - // Compile annotations - let mut annotations_flag = bytecode::MakeFunctionFlags::empty(); - let num_annotations = self.visit_annotations(parameters, returns)?; - if num_annotations > 0 { - annotations_flag = bytecode::MakeFunctionFlags::ANNOTATIONS; - emit!( - self, - Instruction::BuildMap { - size: num_annotations, - } - ); - } + // Compile annotations as closure (PEP 649) + let annotations_flag = if self.compile_annotations_closure(name, parameters, returns)? { + bytecode::MakeFunctionFlags::ANNOTATE + } else { + bytecode::MakeFunctionFlags::empty() + }; // Compile function body let final_funcflags = funcflags | annotations_flag; @@ -3543,10 +3967,14 @@ impl Compiler { fn get_ref_type(&self, name: &str) -> Result { let table = self.symbol_table_stack.last().unwrap(); - // Special handling for __class__ and __classdict__ in class scope + // Special handling for __class__, __classdict__, and __conditional_annotations__ in class scope // This should only apply when we're actually IN a class body, // not when we're in a method nested inside a class. - if table.typ == CompilerScope::Class && (name == "__class__" || name == "__classdict__") { + if table.typ == CompilerScope::Class + && (name == "__class__" + || name == "__classdict__" + || name == "__conditional_annotations__") + { return Ok(SymbolScope::Cell); } match table.lookup(name) { @@ -3669,6 +4097,16 @@ impl Compiler { ); } + // Set __annotate__ closure if present (PEP 649) + if flags.contains(bytecode::MakeFunctionFlags::ANNOTATE) { + emit!( + self, + Instruction::SetFunctionAttribute { + attr: bytecode::MakeFunctionFlags::ANNOTATE + } + ); + } + // Set kwdefaults if present if flags.contains(bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS) { emit!( @@ -3799,9 +4237,31 @@ impl Compiler { emit!(self, Instruction::StoreName(dunder_type_params)); } - // Setup annotations if needed + // PEP 649: Initialize __classdict__ cell for class annotation scope + if self.current_symbol_table().needs_classdict { + let locals_name = self.name("locals"); + emit!(self, Instruction::LoadName(locals_name)); + emit!(self, Instruction::PushNull); + emit!(self, Instruction::Call { nargs: 0 }); + let classdict_idx = self.get_cell_var_index("__classdict__")?; + emit!(self, Instruction::StoreDeref(classdict_idx)); + } + + // Handle class annotations based on future_annotations flag if Self::find_ann(body) { - emit!(self, Instruction::SetupAnnotations); + if self.future_annotations { + // PEP 563: Initialize __annotations__ dict for class + emit!(self, Instruction::SetupAnnotations); + } else { + // PEP 649: Initialize __conditional_annotations__ set if needed for class + if self.current_symbol_table().has_conditional_annotations { + emit!(self, Instruction::BuildSet { size: 0 }); + self.store_name("__conditional_annotations__")?; + } + + // PEP 649: Generate __annotate__ function for class annotations + self.compile_module_annotate(body)?; + } } // 3. Compile the class body @@ -5457,31 +5917,55 @@ impl Compiler { target: &Expr, annotation: &Expr, value: Option<&Expr>, + simple: bool, ) -> CompileResult<()> { + // Perform the actual assignment first if let Some(value) = value { self.compile_expression(value)?; self.compile_store(target)?; } - // Annotations are only evaluated in a module or class. - if self.ctx.in_func() { - return Ok(()); - } - - // Compile annotation: - self.compile_annotation(annotation)?; - - if let Expr::Name(ExprName { id, .. }) = &target { - // Store as dict entry in __annotations__ dict: - let annotations = self.name("__annotations__"); - emit!(self, Instruction::LoadName(annotations)); - self.emit_load_const(ConstantData::Str { - value: self.mangle(id.as_str()).into_owned().into(), - }); - emit!(self, Instruction::StoreSubscr); - } else { - // Drop annotation if not assigned to simple identifier. - emit!(self, Instruction::PopTop); + // If we have a simple name in module or class scope, store annotation + if simple + && !self.ctx.in_func() + && let Expr::Name(ExprName { id, .. }) = target + { + if self.future_annotations { + // PEP 563: Store stringified annotation directly to __annotations__ + // Compile annotation as string + self.compile_annotation(annotation)?; + // Load __annotations__ + let annotations_name = self.name("__annotations__"); + emit!(self, Instruction::LoadName(annotations_name)); + // Load the variable name + self.emit_load_const(ConstantData::Str { + value: self.mangle(id.as_str()).into_owned().into(), + }); + // Store: __annotations__[name] = annotation + emit!(self, Instruction::StoreSubscr); + } else { + // PEP 649: Handle conditional annotations + if self.current_symbol_table().has_conditional_annotations { + // Determine if this annotation is conditional + let is_module = self.current_symbol_table().typ == CompilerScope::Module; + let is_conditional = is_module || self.in_conditional_block; + + if is_conditional { + // Get the current annotation index and increment + let annotation_index = self.next_conditional_annotation_index; + self.next_conditional_annotation_index += 1; + + // Add index to __conditional_annotations__ set + let cond_annotations_name = self.name("__conditional_annotations__"); + emit!(self, Instruction::LoadName(cond_annotations_name)); + self.emit_load_const(ConstantData::Integer { + value: annotation_index.into(), + }); + emit!(self, Instruction::SetAdd { i: 0_u32 }); + emit!(self, Instruction::PopTop); + } + } + } } Ok(()) @@ -6100,10 +6584,7 @@ impl Compiler { in_async_scope: false, }; - self.current_code_info() - .metadata - .consts - .insert_full(ConstantData::None); + // Lambda cannot have docstrings, so no None is added to co_consts self.compile_expression(body)?; self.emit_return_value(); diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index e8eb3c7fee3..0bc76b897dc 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -63,6 +63,17 @@ pub struct SymbolTable { /// Whether this comprehension scope should be inlined (PEP 709) /// True for list/set/dict comprehensions in non-generator expressions pub comp_inlined: bool, + + /// PEP 649: Reference to annotation scope for this block + /// Annotations are compiled as a separate `__annotate__` function + pub annotation_block: Option>, + + /// PEP 649: Whether this scope has conditional annotations + /// (annotations inside if/for/while/etc. blocks or at module level) + pub has_conditional_annotations: bool, + + /// Whether `from __future__ import annotations` is active + pub future_annotations: bool, } impl SymbolTable { @@ -80,6 +91,9 @@ impl SymbolTable { needs_classdict: false, can_see_class_scope: false, comp_inlined: false, + annotation_block: None, + has_conditional_annotations: false, + future_annotations: false, } } @@ -109,6 +123,8 @@ pub enum CompilerScope { Lambda, Comprehension, TypeParams, + /// PEP 649: Annotation scope for deferred evaluation + Annotation, } impl fmt::Display for CompilerScope { @@ -121,9 +137,8 @@ impl fmt::Display for CompilerScope { Self::Lambda => write!(f, "lambda"), Self::Comprehension => write!(f, "comprehension"), Self::TypeParams => write!(f, "type parameter"), + Self::Annotation => write!(f, "annotation"), // TODO missing types from the C implementation - // if self._table.type == _symtable.TYPE_ANNOTATION: - // return "annotation" // if self._table.type == _symtable.TYPE_TYPE_VAR_BOUND: // return "TypeVar bound" // if self._table.type == _symtable.TYPE_TYPE_ALIAS: @@ -246,7 +261,8 @@ impl core::fmt::Debug for SymbolTable { fn analyze_symbol_table(symbol_table: &mut SymbolTable) -> SymbolTableResult { let mut analyzer = SymbolTableAnalyzer::default(); // Discard the newfree set at the top level - it's only needed for propagation - let _newfree = analyzer.analyze_symbol_table(symbol_table)?; + // Pass None for class_entry at top level + let _newfree = analyzer.analyze_symbol_table(symbol_table, None)?; Ok(()) } @@ -269,6 +285,12 @@ fn drop_class_free(symbol_table: &mut SymbolTable, newfree: &mut HashSet if newfree.remove("__classdict__") { symbol_table.needs_classdict = true; } + + // Check if __conditional_annotations__ is in the free variables collected from children + // Remove it from free set - it's handled specially in class scope + if newfree.remove("__conditional_annotations__") { + symbol_table.has_conditional_annotations = true; + } } type SymbolMap = IndexMap; @@ -339,9 +361,11 @@ struct SymbolTableAnalyzer { impl SymbolTableAnalyzer { /// Analyze a symbol table and return the set of free variables. /// See symtable.c analyze_block(). + /// class_entry: PEP 649 - enclosing class symbols for annotation scopes fn analyze_symbol_table( &mut self, symbol_table: &mut SymbolTable, + class_entry: Option<&SymbolMap>, ) -> SymbolTableResult> { let symbols = core::mem::take(&mut symbol_table.symbols); let sub_tables = &mut *symbol_table.sub_tables; @@ -349,15 +373,43 @@ impl SymbolTableAnalyzer { // Collect free variables from all child scopes let mut newfree = HashSet::new(); + let annotation_block = &mut symbol_table.annotation_block; + + // PEP 649: Determine class_entry to pass to children + // If current scope is a class with annotation block that can_see_class_scope, + // we need to pass class symbols to the annotation scope + let is_class = symbol_table.typ == CompilerScope::Class; + + // Clone class symbols if needed for annotation scope (to avoid borrow conflict) + let class_symbols_for_ann = if is_class + && annotation_block + .as_ref() + .is_some_and(|b| b.can_see_class_scope) + { + Some(symbols.clone()) + } else { + None + }; + let mut info = (symbols, symbol_table.typ); self.tables.with_append(&mut info, |list| { let inner_scope = unsafe { &mut *(list as *mut _ as *mut Self) }; // Analyze sub scopes and collect their free variables for sub_table in sub_tables.iter_mut() { - let child_free = inner_scope.analyze_symbol_table(sub_table)?; + // Sub-scopes (functions, nested classes) don't inherit class_entry + let child_free = inner_scope.analyze_symbol_table(sub_table, None)?; // Propagate child's free variables to this scope newfree.extend(child_free); } + // PEP 649: Analyze annotation block if present + if let Some(annotation_table) = annotation_block { + // Pass class symbols to annotation scope if can_see_class_scope + let ann_class_entry = class_symbols_for_ann.as_ref().or(class_entry); + let child_free = + inner_scope.analyze_symbol_table(annotation_table, ann_class_entry)?; + // Propagate annotation's free variables to this scope + newfree.extend(child_free); + } Ok(()) })?; @@ -396,7 +448,7 @@ impl SymbolTableAnalyzer { // Analyze symbols in current scope for symbol in symbol_table.symbols.values_mut() { - self.analyze_symbol(symbol, symbol_table.typ, sub_tables)?; + self.analyze_symbol(symbol, symbol_table.typ, sub_tables, class_entry)?; // Collect free variables from this scope // These will be propagated to the parent scope @@ -420,6 +472,7 @@ impl SymbolTableAnalyzer { symbol: &mut Symbol, st_typ: CompilerScope, sub_tables: &[SymbolTable], + class_entry: Option<&SymbolMap>, ) -> SymbolTableResult { if symbol .flags @@ -439,7 +492,8 @@ impl SymbolTableAnalyzer { // check if the name is already defined in any outer scope // therefore if scope_depth < 2 - || self.found_in_outer_scope(&symbol.name) != Some(SymbolScope::Free) + || self.found_in_outer_scope(&symbol.name, st_typ) + != Some(SymbolScope::Free) { return Err(SymbolTableError { error: format!("no binding for nonlocal '{}' found", symbol.name), @@ -465,11 +519,23 @@ impl SymbolTableAnalyzer { // all is well } SymbolScope::Unknown => { + // PEP 649: Check class_entry first (like analyze_name) + // If name is bound in enclosing class, mark as GlobalImplicit + if let Some(class_symbols) = class_entry + && let Some(class_sym) = class_symbols.get(&symbol.name) + { + // DEF_BOUND && !DEF_NONLOCAL -> GLOBAL_IMPLICIT + if class_sym.is_bound() && class_sym.scope != SymbolScope::Free { + symbol.scope = SymbolScope::GlobalImplicit; + return Ok(()); + } + } + // Try hard to figure out what the scope of this symbol is. let scope = if symbol.is_bound() { self.found_in_inner_scope(sub_tables, &symbol.name, st_typ) .unwrap_or(SymbolScope::Local) - } else if let Some(scope) = self.found_in_outer_scope(&symbol.name) { + } else if let Some(scope) = self.found_in_outer_scope(&symbol.name, st_typ) { scope } else if self.tables.is_empty() { // Don't make assumptions when we don't know. @@ -485,18 +551,40 @@ impl SymbolTableAnalyzer { Ok(()) } - fn found_in_outer_scope(&mut self, name: &str) -> Option { + fn found_in_outer_scope(&mut self, name: &str, st_typ: CompilerScope) -> Option { let mut decl_depth = None; for (i, (symbols, typ)) in self.tables.iter().rev().enumerate() { if matches!(typ, CompilerScope::Module) - || matches!(typ, CompilerScope::Class if name != "__class__") + || matches!(typ, CompilerScope::Class if name != "__class__" && name != "__classdict__" && name != "__conditional_annotations__") { continue; } - // __class__ is implicitly declared in class scope - // This handles the case where super() is called in a nested class method - if name == "__class__" && matches!(typ, CompilerScope::Class) { + // PEP 649: Annotation scope is conceptually a sibling of the function, + // not a child. Skip the immediate parent function scope when looking + // for outer variables from annotation scope. + if st_typ == CompilerScope::Annotation + && i == 0 + && matches!( + typ, + CompilerScope::Function | CompilerScope::AsyncFunction | CompilerScope::Lambda + ) + { + continue; + } + + // __class__ and __classdict__ are implicitly declared in class scope + // This handles the case where nested scopes reference them + if (name == "__class__" || name == "__classdict__") + && matches!(typ, CompilerScope::Class) + { + decl_depth = Some(i); + break; + } + + // __conditional_annotations__ is implicitly declared in class scope + // for classes with conditional annotations + if name == "__conditional_annotations__" && matches!(typ, CompilerScope::Class) { decl_depth = Some(i); break; } @@ -657,6 +745,13 @@ impl SymbolTableAnalyzer { location: None, }); } + CompilerScope::Annotation => { + // Named expression is not allowed in annotation scope + return Err(SymbolTableError { + error: "named expression cannot be used within an annotation".to_string(), + location: None, + }); + } } Ok(()) } @@ -695,6 +790,8 @@ struct SymbolTableBuilder { in_comp_inner_loop_target: bool, // Scope info for error messages (e.g., "a TypeVar bound") scope_info: Option<&'static str>, + // PEP 649: Track if we're inside a conditional block (if/for/while/etc.) + in_conditional_block: bool, } /// Enum to indicate in what mode an expression @@ -723,6 +820,7 @@ impl SymbolTableBuilder { in_type_alias: false, in_comp_inner_loop_target: false, scope_info: None, + in_conditional_block: false, }; this.enter_scope("top", CompilerScope::Module, 0); this @@ -733,6 +831,8 @@ impl SymbolTableBuilder { let mut symbol_table = self.tables.pop().unwrap(); // Save varnames for the top-level module scope symbol_table.varnames = self.current_varnames; + // Propagate future_annotations to the symbol table + symbol_table.future_annotations = self.future_annotations; analyze_symbol_table(&mut symbol_table)?; Ok(symbol_table) } @@ -782,6 +882,79 @@ impl SymbolTableBuilder { self.tables.last_mut().unwrap().sub_tables.push(table); } + /// Enter annotation scope (PEP 649) + /// Creates or reuses the annotation block for the current scope + fn enter_annotation_scope(&mut self, line_number: u32) { + let current = self.tables.last_mut().unwrap(); + let can_see_class_scope = current.typ == CompilerScope::Class; + let has_conditional = current.has_conditional_annotations; + + // Create annotation block if not exists + if current.annotation_block.is_none() { + let mut annotation_table = SymbolTable::new( + "__annotate__".to_owned(), + CompilerScope::Annotation, + line_number, + true, // is_nested + ); + // Annotation scope in class can see class scope + annotation_table.can_see_class_scope = can_see_class_scope; + // Add 'format' parameter + annotation_table.varnames.push("format".to_owned()); + current.annotation_block = Some(Box::new(annotation_table)); + } + + // Take the annotation block and push to stack for processing + let annotation_table = current.annotation_block.take().unwrap(); + self.tables.push(*annotation_table); + self.current_varnames.clear(); + + if can_see_class_scope && !self.future_annotations { + self.add_classdict_freevar(); + // Also add __conditional_annotations__ as free var if parent has conditional annotations + if has_conditional { + self.add_conditional_annotations_freevar(); + } + } + } + + /// Leave annotation scope (PEP 649) + /// Stores the annotation block back to parent instead of sub_tables + fn leave_annotation_scope(&mut self) { + let mut table = self.tables.pop().unwrap(); + // Save the collected varnames to the symbol table + table.varnames = core::mem::take(&mut self.current_varnames); + // Store back to parent's annotation_block (not sub_tables) + let parent = self.tables.last_mut().unwrap(); + parent.annotation_block = Some(Box::new(table)); + } + + fn add_classdict_freevar(&mut self) { + let table = self.tables.last_mut().unwrap(); + let name = "__classdict__"; + let symbol = table + .symbols + .entry(name.to_owned()) + .or_insert_with(|| Symbol::new(name)); + symbol.scope = SymbolScope::Free; + symbol + .flags + .insert(SymbolFlags::REFERENCED | SymbolFlags::FREE_CLASS); + } + + fn add_conditional_annotations_freevar(&mut self) { + let table = self.tables.last_mut().unwrap(); + let name = "__conditional_annotations__"; + let symbol = table + .symbols + .entry(name.to_owned()) + .or_insert_with(|| Symbol::new(name)); + symbol.scope = SymbolScope::Free; + symbol + .flags + .insert(SymbolFlags::REFERENCED | SymbolFlags::FREE_CLASS); + } + fn line_index_start(&self, range: TextRange) -> u32 { self.source_file .to_source_code() @@ -830,15 +1003,58 @@ impl SymbolTableBuilder { } fn scan_annotation(&mut self, annotation: &Expr) -> SymbolTableResult { + let current_scope = self.tables.last().map(|t| t.typ); + + // PEP 649: Check if this is a conditional annotation + // Module-level: always conditional (module may be partially executed) + // Class-level: conditional only when inside if/for/while/etc. + if !self.future_annotations { + let is_conditional = matches!(current_scope, Some(CompilerScope::Module)) + || (matches!(current_scope, Some(CompilerScope::Class)) + && self.in_conditional_block); + + if is_conditional && !self.tables.last().unwrap().has_conditional_annotations { + self.tables.last_mut().unwrap().has_conditional_annotations = true; + // Register __conditional_annotations__ symbol in the scope (USE flag, not DEF) + self.register_name( + "__conditional_annotations__", + SymbolUsage::Used, + annotation.range(), + )?; + } + } + + // Create annotation scope for deferred evaluation + let line_number = self.line_index_start(annotation.range()); + self.enter_annotation_scope(line_number); + if self.future_annotations { - Ok(()) - } else { + // PEP 563: annotations are stringified at compile time + // Don't scan expression - symbols would fail to resolve + // Just create the annotation_block structure + self.leave_annotation_scope(); + return Ok(()); + } + + // PEP 649: scan expression for symbol references + // Class annotations are evaluated in class locals (not module globals) + let was_in_annotation = self.in_annotation; + self.in_annotation = true; + let result = self.scan_expression(annotation, ExpressionContext::Load); + self.in_annotation = was_in_annotation; + + self.leave_annotation_scope(); + + // Module scope: re-scan to register symbols (builtins like str, int) + // Class scope: do NOT re-scan to preserve class-local symbol resolution + if matches!(current_scope, Some(CompilerScope::Module)) { let was_in_annotation = self.in_annotation; self.in_annotation = true; - let result = self.scan_expression(annotation, ExpressionContext::Load); + let _ = self.scan_expression(annotation, ExpressionContext::Load); self.in_annotation = was_in_annotation; - result } + + result } fn scan_statement(&mut self, statement: &Stmt) -> SymbolTableResult { @@ -873,9 +1089,23 @@ impl SymbolTableBuilder { }) => { self.scan_decorators(decorator_list, ExpressionContext::Load)?; self.register_ident(name, SymbolUsage::Assigned)?; - if let Some(expression) = returns { - self.scan_annotation(expression)?; - } + + // Save the parent's annotation_block before scanning function annotations, + // so function annotations don't interfere with parent scope annotations. + // This applies to both class scope (methods) and module scope (top-level functions). + let parent_scope_typ = self.tables.last().map(|t| t.typ); + let should_save_annotation_block = matches!( + parent_scope_typ, + Some(CompilerScope::Class) | Some(CompilerScope::Module) + ); + let saved_annotation_block = if should_save_annotation_block { + self.tables.last_mut().unwrap().annotation_block.take() + } else { + None + }; + + // For generic functions, enter type_param block FIRST so that + // annotation scopes are nested inside and can see type parameters. if let Some(type_params) = type_params { self.enter_type_param_block( &format!("", name.as_str()), @@ -883,16 +1113,28 @@ impl SymbolTableBuilder { )?; self.scan_type_params(type_params)?; } + let has_return_annotation = if let Some(expression) = returns { + self.scan_annotation(expression)?; + true + } else { + false + }; self.enter_scope_with_parameters( name.as_str(), parameters, self.line_index_start(*range), + has_return_annotation, )?; self.scan_statements(body)?; self.leave_scope(); if type_params.is_some() { self.leave_scope(); } + + // Restore parent's annotation_block after processing the function + if let Some(block) = saved_annotation_block { + self.tables.last_mut().unwrap().annotation_block = Some(block); + } } Stmt::ClassDef(StmtClassDef { name, @@ -915,6 +1157,10 @@ impl SymbolTableBuilder { CompilerScope::Class, self.line_index_start(*range), ); + // Reset in_conditional_block for new class scope + // (each scope has its own conditional context) + let saved_in_conditional = self.in_conditional_block; + self.in_conditional_block = false; let prev_class = self.class_name.replace(name.to_string()); self.register_name("__module__", SymbolUsage::Assigned, *range)?; self.register_name("__qualname__", SymbolUsage::Assigned, *range)?; @@ -922,6 +1168,7 @@ impl SymbolTableBuilder { self.register_name("__class__", SymbolUsage::Assigned, *range)?; self.scan_statements(body)?; self.leave_scope(); + self.in_conditional_block = saved_in_conditional; self.class_name = prev_class; if let Some(arguments) = arguments { self.scan_expressions(&arguments.args, ExpressionContext::Load)?; @@ -945,6 +1192,9 @@ impl SymbolTableBuilder { .. }) => { self.scan_expression(test, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; for elif in elif_else_clauses { if let Some(test) = &elif.test { @@ -952,6 +1202,7 @@ impl SymbolTableBuilder { } self.scan_statements(&elif.body)?; } + self.in_conditional_block = saved_in_conditional_block; } Stmt::For(StmtFor { target, @@ -962,15 +1213,23 @@ impl SymbolTableBuilder { }) => { self.scan_expression(target, ExpressionContext::Store)?; self.scan_expression(iter, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; self.scan_statements(orelse)?; + self.in_conditional_block = saved_in_conditional_block; } Stmt::While(StmtWhile { test, body, orelse, .. }) => { self.scan_expression(test, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; self.scan_statements(orelse)?; + self.in_conditional_block = saved_in_conditional_block; } Stmt::Break(_) | Stmt::Continue(_) | Stmt::Pass(_) => { // No symbols here. @@ -1037,6 +1296,21 @@ impl SymbolTableBuilder { match &**target { Expr::Name(ast::ExprName { id, .. }) if *simple => { self.register_name(id.as_str(), SymbolUsage::AnnotationAssigned, *range)?; + // PEP 649: Register annotate function in module/class scope + let current_scope = self.tables.last().map(|t| t.typ); + match current_scope { + Some(CompilerScope::Module) => { + self.register_name("__annotate__", SymbolUsage::Assigned, *range)?; + } + Some(CompilerScope::Class) => { + self.register_name( + "__annotate_func__", + SymbolUsage::Assigned, + *range, + )?; + } + _ => {} + } } _ => { self.scan_expression(target, ExpressionContext::Store)?; @@ -1054,7 +1328,11 @@ impl SymbolTableBuilder { self.scan_expression(expression, ExpressionContext::Store)?; } } + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; + self.in_conditional_block = saved_in_conditional_block; } Stmt::Try(StmtTry { body, @@ -1063,6 +1341,9 @@ impl SymbolTableBuilder { finalbody, .. }) => { + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; for handler in handlers { let ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { @@ -1081,9 +1362,13 @@ impl SymbolTableBuilder { } self.scan_statements(orelse)?; self.scan_statements(finalbody)?; + self.in_conditional_block = saved_in_conditional_block; } Stmt::Match(StmtMatch { subject, cases, .. }) => { self.scan_expression(subject, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; for case in cases { self.scan_pattern(&case.pattern)?; if let Some(guard) = &case.guard { @@ -1091,6 +1376,7 @@ impl SymbolTableBuilder { } self.scan_statements(&case.body)?; } + self.in_conditional_block = saved_in_conditional_block; } Stmt::Raise(StmtRaise { exc, cause, .. }) => { if let Some(expression) = exc { @@ -1412,6 +1698,7 @@ impl SymbolTableBuilder { "lambda", parameters, self.line_index_start(expression.range()), + false, // lambdas have no return annotation )?; } else { self.enter_scope( @@ -1774,6 +2061,7 @@ impl SymbolTableBuilder { name: &str, parameters: &Parameters, line_number: u32, + has_return_annotation: bool, ) -> SymbolTableResult { // Evaluate eventual default parameters: for default in parameters @@ -1811,8 +2099,40 @@ impl SymbolTableBuilder { self.scan_annotation(annotation)?; } + // Check if this function has any annotations (parameter or return) + let has_param_annotations = parameters + .posonlyargs + .iter() + .chain(parameters.args.iter()) + .chain(parameters.kwonlyargs.iter()) + .any(|p| p.parameter.annotation.is_some()) + || parameters + .vararg + .as_ref() + .is_some_and(|p| p.annotation.is_some()) + || parameters + .kwarg + .as_ref() + .is_some_and(|p| p.annotation.is_some()); + + let has_any_annotations = has_param_annotations || has_return_annotation; + + // Take annotation_block if this function has any annotations. + // When in class scope, the class's annotation_block was saved before scanning + // function annotations, so the current annotation_block belongs to this function. + let annotation_block = if has_any_annotations { + self.tables.last_mut().unwrap().annotation_block.take() + } else { + None + }; + self.enter_scope(name, CompilerScope::Function, line_number); + // Move annotation_block to function scope only if we have one + if let Some(block) = annotation_block { + self.tables.last_mut().unwrap().annotation_block = Some(block); + } + // Fill scope with parameter names: self.scan_parameters(¶meters.posonlyargs)?; self.scan_parameters(¶meters.args)?; @@ -1841,6 +2161,20 @@ impl SymbolTableBuilder { .to_source_code() .source_location(range.start(), PositionEncoding::Utf8); let location = Some(location); + + // Check for forbidden names like __debug__ + if name == "__debug__" + && matches!( + role, + SymbolUsage::Parameter | SymbolUsage::AnnotationParameter | SymbolUsage::Assigned + ) + { + return Err(SymbolTableError { + error: "cannot assign to __debug__".to_owned(), + location, + }); + } + let scope_depth = self.tables.len(); let table = self.tables.last_mut().unwrap(); diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs index a0054b28877..c59a64fea3d 100644 --- a/crates/compiler-core/src/bytecode.rs +++ b/crates/compiler-core/src/bytecode.rs @@ -290,13 +290,16 @@ pub struct CodeObject { bitflags! { #[derive(Copy, Clone, Debug, PartialEq)] - pub struct CodeFlags: u16 { + pub struct CodeFlags: u32 { const OPTIMIZED = 0x0001; const NEWLOCALS = 0x0002; const VARARGS = 0x0004; const VARKEYWORDS = 0x0008; const GENERATOR = 0x0020; const COROUTINE = 0x0080; + /// If a code object represents a function and has a docstring, + /// this bit is set and the first item in co_consts is the docstring. + const HAS_DOCSTRING = 0x4000000; } } diff --git a/crates/compiler-core/src/bytecode/oparg.rs b/crates/compiler-core/src/bytecode/oparg.rs index c662a0a9260..3130da59750 100644 --- a/crates/compiler-core/src/bytecode/oparg.rs +++ b/crates/compiler-core/src/bytecode/oparg.rs @@ -327,6 +327,8 @@ bitflags! { const KW_ONLY_DEFAULTS = 0x04; const DEFAULTS = 0x08; const TYPE_PARAMS = 0x10; + /// PEP 649: __annotate__ function closure (instead of __annotations__ dict) + const ANNOTATE = 0x20; } } diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index 5b528fe7e50..d9bf368ecab 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -202,7 +202,7 @@ pub fn deserialize_code( }) .collect::>>()?; - let flags = CodeFlags::from_bits_truncate(rdr.read_u16()?); + let flags = CodeFlags::from_bits_truncate(rdr.read_u32()?); let posonlyarg_count = rdr.read_u32()?; let arg_count = rdr.read_u32()?; @@ -660,7 +660,7 @@ pub fn serialize_code(buf: &mut W, code: &CodeObject) buf.write_u32(end.character_offset.to_zero_indexed() as _); } - buf.write_u16(code.flags.bits()); + buf.write_u32(code.flags.bits()); buf.write_u32(code.posonlyarg_count); buf.write_u32(code.arg_count); diff --git a/crates/derive-impl/src/pyclass.rs b/crates/derive-impl/src/pyclass.rs index a81a7bacbad..57cbf67de5a 100644 --- a/crates/derive-impl/src/pyclass.rs +++ b/crates/derive-impl/src/pyclass.rs @@ -778,36 +778,10 @@ pub(crate) fn impl_pyexception_impl(attr: PunctuatedNestedMeta, item: Item) -> R } }; - // We need this method, because of how `CPython` copies `__init__` - // from `BaseException` in `SimpleExtendsException` macro. - // See: `(initproc)BaseException_init` - // spell-checker:ignore initproc - let slot_init = if with_contains(&with_items, "Initializer") { - quote!() - } else { - with_items.push(Ident::new("Initializer", Span::call_site())); - quote! { - impl ::rustpython_vm::types::Initializer for #self_ty { - type Args = ::rustpython_vm::function::FuncArgs; - - fn slot_init( - zelf: ::rustpython_vm::PyObjectRef, - args: ::rustpython_vm::function::FuncArgs, - vm: &::rustpython_vm::VirtualMachine, - ) -> ::rustpython_vm::PyResult<()> { - ::Base::slot_init(zelf, args, vm) - } - - fn init( - _zelf: ::rustpython_vm::PyRef, - _args: Self::Args, - _vm: &::rustpython_vm::VirtualMachine - ) -> ::rustpython_vm::PyResult<()> { - unreachable!("slot_init is defined") - } - } - } - }; + // SimpleExtendsException: inherits BaseException_init from the base class via MRO. + // Only exceptions that explicitly specify `with(Initializer)` will have + // their own __init__ in __dict__. + let slot_init = quote!(); let extra_attrs_tokens = if extra_attrs.is_empty() { quote!() diff --git a/crates/jit/tests/common.rs b/crates/jit/tests/common.rs index eabf27471b9..da324088d32 100644 --- a/crates/jit/tests/common.rs +++ b/crates/jit/tests/common.rs @@ -63,6 +63,72 @@ impl From for StackValue { } } +/// Extract annotations from an annotate function's bytecode. +/// The annotate function uses BUILD_MAP with key-value pairs loaded before it. +/// Keys are parameter names (from LOAD_CONST), values are type names (from LOAD_NAME/LOAD_GLOBAL). +fn extract_annotations_from_annotate_code(code: &CodeObject) -> HashMap { + let mut annotations = HashMap::new(); + let mut stack: Vec<(bool, usize)> = Vec::new(); // (is_const, index) + let mut op_arg_state = OpArgState::default(); + + for &word in code.instructions.iter() { + let (instruction, arg) = op_arg_state.get(word); + + match instruction { + Instruction::LoadConst { idx } => { + stack.push((true, idx.get(arg) as usize)); + } + Instruction::LoadName(idx) | Instruction::LoadGlobal(idx) => { + stack.push((false, idx.get(arg) as usize)); + } + Instruction::BuildMap { size, .. } => { + let count = size.get(arg) as usize; + // Stack has key-value pairs in order: k1, v1, k2, v2, ... + // So we need count * 2 items from the stack + let start = stack.len().saturating_sub(count * 2); + let pairs: Vec<_> = stack.drain(start..).collect(); + + for chunk in pairs.chunks(2) { + if chunk.len() == 2 { + let (key_is_const, key_idx) = chunk[0]; + let (_val_is_const, val_idx) = chunk[1]; + + // Key should be a const string (parameter name) + if key_is_const + && let ConstantData::Str { value } = &code.constants[key_idx] + { + let param_name = value.to_string_lossy().into_owned(); + // Value should be a name (type name) + if let Some(type_name) = code.names.get(val_idx) { + annotations + .insert(param_name, StackValue::String(type_name.clone())); + } + } + } + } + // Return after processing BUILD_MAP - we got our annotations + return annotations; + } + Instruction::Resume { .. } + | Instruction::LoadFast(_) + | Instruction::CompareOp { .. } + | Instruction::ExtendedArg => { + // Ignore these instructions for annotation extraction + } + Instruction::ReturnValue | Instruction::ReturnConst { .. } => { + // End of function - return what we have + return annotations; + } + _ => { + // For other instructions, clear the stack tracking as we don't understand the effect + stack.clear(); + } + } + } + + annotations +} + pub struct StackMachine { stack: Vec, locals: HashMap, @@ -92,6 +158,9 @@ impl StackMachine { names: &[String], ) -> ControlFlow<()> { match instruction { + Instruction::Resume { .. } => { + // No-op for JIT tests - just marks function entry point + } Instruction::LoadConst { idx } => { let idx = idx.get(arg); self.stack.push(constants[idx as usize].clone().into()) @@ -143,13 +212,31 @@ impl StackMachine { }; let attr_value = self.stack.pop().expect("Expected attribute value on stack"); - // For now, we only handle ANNOTATIONS flag in JIT tests - if attr - .get(arg) + let flags = attr.get(arg); + + // Handle ANNOTATE flag (PEP 649 style - Python 3.14+) + // The attr_value is a function that returns annotations when called + if flags.contains(rustpython_compiler_core::bytecode::MakeFunctionFlags::ANNOTATE) { + if let StackValue::Function(annotate_func) = attr_value { + // Parse the annotate function's bytecode to extract annotations + // The pattern is: LOAD_CONST (key), LOAD_NAME (value), ... BUILD_MAP + let annotate_code = &annotate_func.code; + let annotations = extract_annotations_from_annotate_code(annotate_code); + + let updated_func = Function { + code: func.code, + annotations, + }; + self.stack.push(StackValue::Function(updated_func)); + } else { + panic!("Expected annotate function for ANNOTATE flag"); + } + } + // Handle old ANNOTATIONS flag (Python 3.12 style) + else if flags .contains(rustpython_compiler_core::bytecode::MakeFunctionFlags::ANNOTATIONS) { if let StackValue::Map(annotations) = attr_value { - // Update function's annotations let updated_func = Function { code: func.code, annotations, @@ -160,7 +247,6 @@ impl StackMachine { } } else { // For other attributes, just push the function back unchanged - // (since JIT tests mainly care about type annotations) self.stack.push(StackValue::Function(func)); } } diff --git a/crates/stdlib/src/hashlib.rs b/crates/stdlib/src/hashlib.rs index 2da47ceb740..261663c8564 100644 --- a/crates/stdlib/src/hashlib.rs +++ b/crates/stdlib/src/hashlib.rs @@ -11,7 +11,7 @@ pub mod _hashlib { class::StaticType, convert::ToPyObject, function::{ArgBytesLike, ArgStrOrBytesLike, FuncArgs, OptionalArg}, - types::{Constructor, Initializer, Representable}, + types::{Constructor, Representable}, }; use blake2::{Blake2b512, Blake2s256}; use digest::{DynDigest, core_api::BlockSizeUser}; diff --git a/crates/stdlib/src/ssl/error.rs b/crates/stdlib/src/ssl/error.rs index d77910f6aa1..6219eff41b5 100644 --- a/crates/stdlib/src/ssl/error.rs +++ b/crates/stdlib/src/ssl/error.rs @@ -7,7 +7,7 @@ pub(crate) mod ssl_error { use crate::vm::{ Py, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyBaseException, PyOSError, PyStrRef}, - types::{Constructor, Initializer}, + types::Constructor, }; // Error type constants - exposed as pyattr and available for internal use diff --git a/crates/venvlauncher/src/main.rs b/crates/venvlauncher/src/main.rs index fe147ce7ff3..7087e791e37 100644 --- a/crates/venvlauncher/src/main.rs +++ b/crates/venvlauncher/src/main.rs @@ -114,12 +114,12 @@ mod tests { let cfg_path = temp_dir.join("test_pyvenv.cfg"); let mut file = fs::File::create(&cfg_path).unwrap(); - writeln!(file, "home = C:\\Python313").unwrap(); + writeln!(file, "home = C:\\Python314").unwrap(); writeln!(file, "include-system-site-packages = false").unwrap(); - writeln!(file, "version = 3.13.0").unwrap(); + writeln!(file, "version = 3.14.0").unwrap(); let home = read_home(&cfg_path).unwrap(); - assert_eq!(home, "C:\\Python313"); + assert_eq!(home, "C:\\Python314"); fs::remove_file(&cfg_path).unwrap(); } diff --git a/crates/vm/src/builtins/bytearray.rs b/crates/vm/src/builtins/bytearray.rs index 0f9dce7230b..5a3aee59072 100644 --- a/crates/vm/src/builtins/bytearray.rs +++ b/crates/vm/src/builtins/bytearray.rs @@ -322,8 +322,8 @@ impl PyByteArray { } #[pyclassmethod] - fn fromhex(cls: PyTypeRef, string: PyStrRef, vm: &VirtualMachine) -> PyResult { - let bytes = PyBytesInner::fromhex(string.as_bytes(), vm)?; + fn fromhex(cls: PyTypeRef, string: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let bytes = PyBytesInner::fromhex_object(string, vm)?; let bytes = vm.ctx.new_bytes(bytes); let args = vec![bytes.into()].into(); PyType::call(&cls, args, vm) diff --git a/crates/vm/src/builtins/bytes.rs b/crates/vm/src/builtins/bytes.rs index 01e67358aae..64e00cdc919 100644 --- a/crates/vm/src/builtins/bytes.rs +++ b/crates/vm/src/builtins/bytes.rs @@ -316,8 +316,8 @@ impl PyBytes { } #[pyclassmethod] - fn fromhex(cls: PyTypeRef, string: PyStrRef, vm: &VirtualMachine) -> PyResult { - let bytes = PyBytesInner::fromhex(string.as_bytes(), vm)?; + fn fromhex(cls: PyTypeRef, string: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let bytes = PyBytesInner::fromhex_object(string, vm)?; let bytes = vm.ctx.new_bytes(bytes).into(); PyType::call(&cls, vec![bytes].into(), vm) } diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs index 85816aabb7c..6507af342c1 100644 --- a/crates/vm/src/builtins/code.rs +++ b/crates/vm/src/builtins/code.rs @@ -152,7 +152,7 @@ pub struct ReplaceArgs { #[pyarg(named, optional)] co_names: OptionalArg>, #[pyarg(named, optional)] - co_flags: OptionalArg, + co_flags: OptionalArg, #[pyarg(named, optional)] co_varnames: OptionalArg>, #[pyarg(named, optional)] @@ -411,7 +411,7 @@ pub struct PyCodeNewArgs { kwonlyargcount: u32, nlocals: u32, stacksize: u32, - flags: u16, + flags: u32, co_code: PyBytesRef, consts: PyTupleRef, names: PyTupleRef, @@ -628,7 +628,7 @@ impl PyCode { } #[pygetset] - const fn co_flags(&self) -> u16 { + const fn co_flags(&self) -> u32 { self.code.flags.bits() } diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index 3d5159e58c1..9297cf07201 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -36,7 +36,8 @@ pub struct PyFunction { name: PyMutex, qualname: PyMutex, type_params: PyMutex, - annotations: PyMutex, + annotations: PyMutex>, + annotate: PyMutex>, module: PyMutex, doc: PyMutex, #[cfg(feature = "jit")] @@ -77,6 +78,17 @@ impl PyFunction { builtins }; + // Get docstring from co_consts[0] if HAS_DOCSTRING flag is set + let doc = if code.code.flags.contains(bytecode::CodeFlags::HAS_DOCSTRING) { + code.code + .constants + .first() + .map(|c| c.as_object().to_owned()) + .unwrap_or_else(|| vm.ctx.none()) + } else { + vm.ctx.none() + }; + let qualname = vm.ctx.new_str(code.qualname.as_str()); let func = Self { code: PyMutex::new(code.clone()), @@ -87,9 +99,10 @@ impl PyFunction { name, qualname: PyMutex::new(qualname), type_params: PyMutex::new(vm.ctx.empty_tuple.clone()), - annotations: PyMutex::new(vm.ctx.new_dict()), + annotations: PyMutex::new(None), + annotate: PyMutex::new(None), module: PyMutex::new(module), - doc: PyMutex::new(vm.ctx.none()), + doc: PyMutex::new(doc), #[cfg(feature = "jit")] jitted_code: OnceCell::new(), }; @@ -364,7 +377,7 @@ impl PyFunction { ))); } }; - *self.annotations.lock() = annotations; + *self.annotations.lock() = Some(annotations); } else if attr == bytecode::MakeFunctionFlags::CLOSURE { // For closure, we need special handling // The closure tuple contains cell objects @@ -388,6 +401,12 @@ impl PyFunction { )) })?; *self.type_params.lock() = type_params; + } else if attr == bytecode::MakeFunctionFlags::ANNOTATE { + // PEP 649: Store the __annotate__ function closure + if !attr_value.is_callable() { + return Err(vm.new_type_error("__annotate__ must be callable".to_owned())); + } + *self.annotate.lock() = Some(attr_value); } else { unreachable!("This is a compiler bug"); } @@ -574,13 +593,103 @@ impl PyFunction { } #[pygetset] - fn __annotations__(&self) -> PyDictRef { - self.annotations.lock().clone() + fn __annotations__(&self, vm: &VirtualMachine) -> PyResult { + // First check if we have cached annotations + { + let annotations = self.annotations.lock(); + if let Some(ref ann) = *annotations { + return Ok(ann.clone()); + } + } + + // Check for callable __annotate__ and clone it before calling + let annotate_fn = { + let annotate = self.annotate.lock(); + if let Some(ref func) = *annotate + && func.is_callable() + { + Some(func.clone()) + } else { + None + } + }; + + // Release locks before calling __annotate__ to avoid deadlock + if let Some(annotate_fn) = annotate_fn { + let one = vm.ctx.new_int(1); + let ann_dict = annotate_fn.call((one,), vm)?; + let ann_dict = ann_dict + .downcast::() + .map_err(|obj| { + vm.new_type_error(format!( + "__annotate__ returned non-dict of type '{}'", + obj.class().name() + )) + })?; + + // Cache the result + *self.annotations.lock() = Some(ann_dict.clone()); + return Ok(ann_dict); + } + + // No __annotate__ or not callable, create empty dict + let new_dict = vm.ctx.new_dict(); + *self.annotations.lock() = Some(new_dict.clone()); + Ok(new_dict) + } + + #[pygetset(setter)] + fn set___annotations__( + &self, + value: PySetterValue>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let annotations = match value { + PySetterValue::Assign(Some(value)) => { + let annotations = value.downcast::().map_err(|_| { + vm.new_type_error("__annotations__ must be set to a dict object") + })?; + Some(annotations) + } + PySetterValue::Assign(None) | PySetterValue::Delete => None, + }; + *self.annotations.lock() = annotations; + + // Clear __annotate__ when __annotations__ is set + *self.annotate.lock() = None; + Ok(()) + } + + #[pygetset] + fn __annotate__(&self, vm: &VirtualMachine) -> PyObjectRef { + self.annotate + .lock() + .clone() + .unwrap_or_else(|| vm.ctx.none()) } #[pygetset(setter)] - fn set___annotations__(&self, annotations: PyDictRef) { - *self.annotations.lock() = annotations + fn set___annotate__( + &self, + value: PySetterValue>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let annotate = match value { + PySetterValue::Assign(Some(value)) => { + if !value.is_callable() { + return Err(vm.new_type_error("__annotate__ must be callable or None")); + } + // Clear cached __annotations__ when __annotate__ is set + *self.annotations.lock() = None; + Some(value) + } + PySetterValue::Assign(None) => None, + PySetterValue::Delete => { + return Err(vm.new_type_error("__annotate__ cannot be deleted")); + } + }; + *self.annotate.lock() = annotate; + Ok(()) } #[pygetset] diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index 9b74b66e3a7..765f5e2454b 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -12,7 +12,8 @@ use crate::{ }, convert::{IntoPyException, ToPyObject, ToPyResult}, function::{ - ArgByteOrder, ArgIntoBool, FuncArgs, OptionalArg, PyArithmeticValue, PyComparisonValue, + ArgByteOrder, ArgIntoBool, FuncArgs, OptionalArg, OptionalOption, PyArithmeticValue, + PyComparisonValue, }, protocol::{PyNumberMethods, handle_bytes_to_int_err}, types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, @@ -286,6 +287,13 @@ impl PyInt { where I: PrimInt + TryFrom<&'a BigInt>, { + // TODO: Python 3.14+: ValueError for negative int to unsigned type + // See stdlib_socket.py socket.htonl(-1) + // + // if I::min_value() == I::zero() && self.as_bigint().sign() == Sign::Minus { + // return Err(vm.new_value_error("Cannot convert negative int".to_owned())); + // } + I::try_from(self.as_bigint()).map_err(|_| { vm.new_overflow_error(format!( "Python int too large to convert to Rust {}", @@ -381,10 +389,10 @@ impl PyInt { #[pymethod] fn __round__( zelf: PyRef, - ndigits: OptionalArg, + ndigits: OptionalOption, vm: &VirtualMachine, ) -> PyResult> { - if let OptionalArg::Present(ndigits) = ndigits { + if let Some(ndigits) = ndigits.flatten() { let ndigits = ndigits.as_bigint(); // round(12345, -2) == 12300 // If precision >= 0, then any integer is already rounded correctly diff --git a/crates/vm/src/builtins/module.rs b/crates/vm/src/builtins/module.rs index faa6e4813fd..27cc7135278 100644 --- a/crates/vm/src/builtins/module.rs +++ b/crates/vm/src/builtins/module.rs @@ -4,7 +4,7 @@ use crate::{ builtins::{PyStrInterned, pystr::AsPyStr}, class::PyClassImpl, convert::ToPyObject, - function::{FuncArgs, PyMethodDef}, + function::{FuncArgs, PyMethodDef, PySetterValue}, types::{GetAttr, Initializer, Representable}, }; @@ -182,6 +182,116 @@ impl PyModule { let attrs = dict.into_iter().map(|(k, _v)| k).collect(); Ok(attrs) } + + #[pygetset] + fn __annotate__(zelf: &Py, vm: &VirtualMachine) -> PyResult { + let dict = zelf.dict(); + // Get __annotate__ from dict; if not present, insert None and return it + // See: module_get_annotate() + if let Some(annotate) = dict.get_item_opt(identifier!(vm, __annotate__), vm)? { + Ok(annotate) + } else { + let none = vm.ctx.none(); + dict.set_item(identifier!(vm, __annotate__), none.clone(), vm)?; + Ok(none) + } + } + + #[pygetset(setter)] + fn set___annotate__( + zelf: &Py, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(value) => { + if !vm.is_none(&value) && !value.is_callable() { + return Err(vm.new_type_error("__annotate__ must be callable or None")); + } + let dict = zelf.dict(); + dict.set_item(identifier!(vm, __annotate__), value.clone(), vm)?; + // Clear __annotations__ if value is not None + if !vm.is_none(&value) { + dict.del_item(identifier!(vm, __annotations__), vm).ok(); + } + Ok(()) + } + PySetterValue::Delete => Err(vm.new_type_error("cannot delete __annotate__ attribute")), + } + } + + #[pygetset] + fn __annotations__(zelf: &Py, vm: &VirtualMachine) -> PyResult { + let dict = zelf.dict(); + + // Check if __annotations__ is already in dict (explicitly set) + if let Some(annotations) = dict.get_item_opt(identifier!(vm, __annotations__), vm)? { + return Ok(annotations); + } + + // Check if module is initializing + let is_initializing = Self::is_initializing(&dict, vm); + + // PEP 649: Get __annotate__ and call it if callable + let annotations = if let Some(annotate) = + dict.get_item_opt(identifier!(vm, __annotate__), vm)? + && annotate.is_callable() + { + // Call __annotate__(1) where 1 is FORMAT_VALUE + let result = annotate.call((1i32,), vm)?; + if !result.class().is(vm.ctx.types.dict_type) { + return Err(vm.new_type_error(format!( + "__annotate__ returned non-dict of type '{}'", + result.class().name() + ))); + } + result + } else { + vm.ctx.new_dict().into() + }; + + // Cache result unless module is initializing + if !is_initializing { + dict.set_item(identifier!(vm, __annotations__), annotations.clone(), vm)?; + } + + Ok(annotations) + } + + /// Check if module is initializing via __spec__._initializing + fn is_initializing(dict: &PyDictRef, vm: &VirtualMachine) -> bool { + if let Ok(Some(spec)) = dict.get_item_opt(vm.ctx.intern_str("__spec__"), vm) + && let Ok(initializing) = spec.get_attr(vm.ctx.intern_str("_initializing"), vm) + { + return initializing.try_to_bool(vm).unwrap_or(false); + } + false + } + + #[pygetset(setter)] + fn set___annotations__( + zelf: &Py, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + let dict = zelf.dict(); + match value { + PySetterValue::Assign(value) => { + dict.set_item(identifier!(vm, __annotations__), value, vm)?; + // Clear __annotate__ from dict + dict.del_item(identifier!(vm, __annotate__), vm).ok(); + Ok(()) + } + PySetterValue::Delete => { + if dict.del_item(identifier!(vm, __annotations__), vm).is_err() { + return Err(vm.new_attribute_error("__annotations__".to_owned())); + } + // Also clear __annotate__ + dict.del_item(identifier!(vm, __annotate__), vm).ok(); + Ok(()) + } + } + } } impl Initializer for PyModule { diff --git a/crates/vm/src/builtins/object.rs b/crates/vm/src/builtins/object.rs index 982e11afd00..eb9d226acb4 100644 --- a/crates/vm/src/builtins/object.rs +++ b/crates/vm/src/builtins/object.rs @@ -244,6 +244,12 @@ fn object_getstate_default(obj: &PyObject, required: bool, vm: &VirtualMachine) let slots = vm.ctx.new_dict(); for i in 0..slot_names_len { let borrowed_names = slot_names.borrow_vec(); + // Check if slotnames changed during iteration + if borrowed_names.len() != slot_names_len { + return Err(vm.new_runtime_error( + "__slotnames__ changed size during iteration".to_owned(), + )); + } let name = borrowed_names[i].downcast_ref::().unwrap(); let Ok(value) = obj.get_attr(name, vm) else { continue; @@ -702,11 +708,13 @@ fn reduce_newobj(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { (newobj, newargs.into()) } else { + // args == NULL with non-empty kwargs is BadInternalCall + let Some(args) = args else { + return Err(vm.new_system_error("bad internal call".to_owned())); + }; // Use copyreg.__newobj_ex__ let newobj = copyreg.get_attr("__newobj_ex__", vm)?; - let args_tuple: PyObjectRef = args - .map(|a| a.into()) - .unwrap_or_else(|| vm.ctx.empty_tuple.clone().into()); + let args_tuple: PyObjectRef = args.into(); let kwargs_dict: PyObjectRef = kwargs .map(|k| k.into()) .unwrap_or_else(|| vm.ctx.new_dict().into()); diff --git a/crates/vm/src/builtins/singletons.rs b/crates/vm/src/builtins/singletons.rs index 61ab1968a45..169104efeb3 100644 --- a/crates/vm/src/builtins/singletons.rs +++ b/crates/vm/src/builtins/singletons.rs @@ -108,11 +108,12 @@ impl PyNotImplemented { impl AsNumber for PyNotImplemented { fn as_number() -> &'static PyNumberMethods { - // TODO: As per https://bugs.python.org/issue35712, using NotImplemented - // in boolean contexts will need to raise a DeprecationWarning in 3.9 - // and, eventually, a TypeError. static AS_NUMBER: PyNumberMethods = PyNumberMethods { - boolean: Some(|_number, _vm| Ok(true)), + boolean: Some(|_number, vm| { + Err(vm.new_type_error( + "NotImplemented should not be used in a boolean context".to_owned(), + )) + }), ..PyNumberMethods::NOT_IMPLEMENTED }; &AS_NUMBER diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index fed9af976f6..829f7d4439a 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -851,6 +851,58 @@ impl PyType { Ok(()) } + #[pygetset] + fn __annotate__(&self, vm: &VirtualMachine) -> PyResult { + if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '__annotate__'", + self.name() + ))); + } + + let mut attrs = self.attributes.write(); + // First try __annotate__, in case that's been set explicitly + if let Some(annotate) = attrs.get(identifier!(vm, __annotate__)).cloned() { + return Ok(annotate); + } + // Then try __annotate_func__ + if let Some(annotate) = attrs.get(identifier!(vm, __annotate_func__)).cloned() { + // TODO: Apply descriptor tp_descr_get if needed + return Ok(annotate); + } + // Set __annotate_func__ = None and return None + let none = vm.ctx.none(); + attrs.insert(identifier!(vm, __annotate_func__), none.clone()); + Ok(none) + } + + #[pygetset(setter)] + fn set___annotate__(&self, value: Option, vm: &VirtualMachine) -> PyResult<()> { + if value.is_none() { + return Err(vm.new_type_error("cannot delete __annotate__ attribute".to_owned())); + } + let value = value.unwrap(); + + if self.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { + return Err(vm.new_type_error(format!( + "cannot set '__annotate__' attribute of immutable type '{}'", + self.name() + ))); + } + + if !vm.is_none(&value) && !value.is_callable() { + return Err(vm.new_type_error("__annotate__ must be callable or None".to_owned())); + } + + let mut attrs = self.attributes.write(); + // Store to __annotate_func__ + attrs.insert(identifier!(vm, __annotate_func__), value.clone()); + // Always clear cached annotations when __annotate__ is updated + attrs.swap_remove(identifier!(vm, __annotations_cache__)); + + Ok(()) + } + #[pygetset] fn __annotations__(&self, vm: &VirtualMachine) -> PyResult { if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { @@ -860,20 +912,37 @@ impl PyType { ))); } - let __annotations__ = identifier!(vm, __annotations__); - let annotations = self.attributes.read().get(__annotations__).cloned(); + // First try __annotations__ (e.g. for "from __future__ import annotations") + let attrs = self.attributes.read(); + if let Some(annotations) = attrs.get(identifier!(vm, __annotations__)).cloned() { + return Ok(annotations); + } + // Then try __annotations_cache__ + if let Some(annotations) = attrs.get(identifier!(vm, __annotations_cache__)).cloned() { + return Ok(annotations); + } + drop(attrs); - let annotations = if let Some(annotations) = annotations { - annotations + // Get __annotate__ and call it if callable + let annotate = self.__annotate__(vm)?; + let annotations = if annotate.is_callable() { + // Call __annotate__(1) where 1 is FORMAT_VALUE + let result = annotate.call((1i32,), vm)?; + if !result.class().is(vm.ctx.types.dict_type) { + return Err(vm.new_type_error(format!( + "__annotate__ returned non-dict of type '{}'", + result.class().name() + ))); + } + result } else { - let annotations: PyObjectRef = vm.ctx.new_dict().into(); - let removed = self - .attributes - .write() - .insert(__annotations__, annotations.clone()); - debug_assert!(removed.is_none()); - annotations + vm.ctx.new_dict().into() }; + + // Cache the result in __annotations_cache__ + self.attributes + .write() + .insert(identifier!(vm, __annotations_cache__), annotations.clone()); Ok(annotations) } @@ -886,21 +955,36 @@ impl PyType { ))); } - let __annotations__ = identifier!(vm, __annotations__); - if let Some(value) = value { - self.attributes.write().insert(__annotations__, value); + let mut attrs = self.attributes.write(); + // conditional update based on __annotations__ presence + let has_annotations = attrs.contains_key(identifier!(vm, __annotations__)); + + if has_annotations { + // If __annotations__ is in dict, update it + if let Some(value) = value { + attrs.insert(identifier!(vm, __annotations__), value); + } else if attrs + .swap_remove(identifier!(vm, __annotations__)) + .is_none() + { + return Err(vm.new_attribute_error("__annotations__".to_owned())); + } + // Also clear __annotations_cache__ + attrs.swap_remove(identifier!(vm, __annotations_cache__)); } else { - self.attributes - .read() - .get(__annotations__) - .cloned() - .ok_or_else(|| { - vm.new_attribute_error(format!( - "'{}' object has no attribute '__annotations__'", - self.name() - )) - })?; + // Otherwise update only __annotations_cache__ + if let Some(value) = value { + attrs.insert(identifier!(vm, __annotations_cache__), value); + } else if attrs + .swap_remove(identifier!(vm, __annotations_cache__)) + .is_none() + { + return Err(vm.new_attribute_error("__annotations__".to_owned())); + } } + // Always clear __annotate_func__ and __annotate__ + attrs.swap_remove(identifier!(vm, __annotate_func__)); + attrs.swap_remove(identifier!(vm, __annotate__)); Ok(()) } diff --git a/crates/vm/src/bytes_inner.rs b/crates/vm/src/bytes_inner.rs index bb5db442c35..7e1c1c2220c 100644 --- a/crates/vm/src/bytes_inner.rs +++ b/crates/vm/src/bytes_inner.rs @@ -423,12 +423,13 @@ impl PyBytesInner { pub fn fromhex(bytes: &[u8], vm: &VirtualMachine) -> PyResult> { let mut iter = bytes.iter().enumerate(); - let mut bytes: Vec = Vec::with_capacity(bytes.len() / 2); - let i = loop { + let mut result: Vec = Vec::with_capacity(bytes.len() / 2); + // None means odd number of hex digits, Some(i) means invalid char at position i + let invalid_char: Option = loop { let (i, &b) = match iter.next() { Some(val) => val, None => { - return Ok(bytes); + return Ok(result); } }; @@ -440,27 +441,49 @@ impl PyBytesInner { b'0'..=b'9' => b - b'0', b'a'..=b'f' => 10 + b - b'a', b'A'..=b'F' => 10 + b - b'A', - _ => break i, + _ => break Some(i), }; let (i, b) = match iter.next() { Some(val) => val, - None => break i + 1, + None => break None, // odd number of hex digits }; let bot = match b { b'0'..=b'9' => b - b'0', b'a'..=b'f' => 10 + b - b'a', b'A'..=b'F' => 10 + b - b'A', - _ => break i, + _ => break Some(i), }; - bytes.push((top << 4) + bot); + result.push((top << 4) + bot); }; - Err(vm.new_value_error(format!( - "non-hexadecimal number found in fromhex() arg at position {i}" - ))) + match invalid_char { + None => Err(vm.new_value_error( + "fromhex() arg must contain an even number of hexadecimal digits".to_owned(), + )), + Some(i) => Err(vm.new_value_error(format!( + "non-hexadecimal number found in fromhex() arg at position {i}" + ))), + } + } + + /// Parse hex string from str or bytes-like object + pub fn fromhex_object(string: PyObjectRef, vm: &VirtualMachine) -> PyResult> { + if let Some(s) = string.downcast_ref::() { + Self::fromhex(s.as_bytes(), vm) + } else if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, &string) { + let borrowed = buffer.as_contiguous().ok_or_else(|| { + vm.new_buffer_error("fromhex() requires a contiguous buffer".to_owned()) + })?; + Self::fromhex(&borrowed, vm) + } else { + Err(vm.new_type_error(format!( + "fromhex() argument must be str or bytes-like, not {}", + string.class().name() + ))) + } } #[inline] diff --git a/crates/vm/src/exception_group.rs b/crates/vm/src/exception_group.rs index a55273480f6..7ad27c078af 100644 --- a/crates/vm/src/exception_group.rs +++ b/crates/vm/src/exception_group.rs @@ -348,8 +348,18 @@ pub(super) mod types { impl Initializer for PyBaseExceptionGroup { type Args = FuncArgs; - fn slot_init(_zelf: PyObjectRef, _args: FuncArgs, _vm: &VirtualMachine) -> PyResult<()> { - // No-op: __new__ already set up the correct args (message, exceptions_tuple) + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // BaseExceptionGroup_init: no kwargs allowed + if !args.kwargs.is_empty() { + return Err(vm.new_type_error(format!( + "{} does not take keyword arguments", + zelf.class().name() + ))); + } + // Do NOT call PyBaseException::slot_init here. + // slot_new already set args to (message, exceptions_tuple). + // Calling base init would overwrite with original args (message, exceptions_list). + let _ = (zelf, args, vm); Ok(()) } diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index a2b08655822..dc25b6d2c03 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -1086,15 +1086,16 @@ fn syntax_error_set_msg( } fn system_exit_code(exc: PyBaseExceptionRef) -> Option { - exc.args.read().first().map(|code| { - match_class!(match code { - ref tup @ PyTuple => match tup.as_slice() { - [x] => x.clone(), - _ => code.clone(), - }, - other => other.clone(), - }) - }) + // SystemExit.code based on args length: + // - size == 0: code is None + // - size == 1: code is args[0] + // - size > 1: code is args (the whole tuple) + let args = exc.args.read(); + match args.len() { + 0 => None, + 1 => Some(args.first().unwrap().clone()), + _ => Some(args.as_object().to_owned()), + } } #[cfg(feature = "serde")] @@ -1255,7 +1256,7 @@ pub(super) mod types { }, convert::ToPyObject, convert::ToPyResult, - function::{ArgBytesLike, FuncArgs}, + function::{ArgBytesLike, FuncArgs, KwArgs}, types::{Constructor, Initializer}, }; use crossbeam_utils::atomic::AtomicCell; @@ -1393,11 +1394,29 @@ pub(super) mod types { pub(super) args: PyRwLock, } - #[pyexception(name, base = PyBaseException, ctx = "system_exit", impl)] + #[pyexception(name, base = PyBaseException, ctx = "system_exit")] #[derive(Debug)] #[repr(transparent)] pub struct PySystemExit(PyBaseException); + // SystemExit_init: has its own __init__ that sets the code attribute + #[pyexception(with(Initializer))] + impl PySystemExit {} + + impl Initializer for PySystemExit { + type Args = FuncArgs; + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // Call BaseException_init first (handles args) + PyBaseException::slot_init(zelf, args, vm) + // Note: code is computed dynamically via system_exit_code getter + // so we don't need to set it here explicitly + } + + fn init(_zelf: PyRef, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + #[pyexception(name, base = PyBaseException, ctx = "generator_exit", impl)] #[derive(Debug)] #[repr(transparent)] @@ -1474,16 +1493,25 @@ pub(super) mod types { type Args = FuncArgs; fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - zelf.set_attr( - "name", - vm.unwrap_or_none(args.kwargs.get("name").cloned()), - vm, - )?; - zelf.set_attr( - "obj", - vm.unwrap_or_none(args.kwargs.get("obj").cloned()), - vm, - )?; + // Only 'name' and 'obj' kwargs are allowed + let mut kwargs = args.kwargs.clone(); + let name = kwargs.swap_remove("name"); + let obj = kwargs.swap_remove("obj"); + + // Reject unknown kwargs + if let Some(invalid_key) = kwargs.keys().next() { + return Err(vm.new_type_error(format!( + "AttributeError() got an unexpected keyword argument '{invalid_key}'" + ))); + } + + // Pass args without kwargs to BaseException_init + let base_args = FuncArgs::new(args.args.clone(), KwArgs::default()); + PyBaseException::slot_init(zelf.clone(), base_args, vm)?; + + // Set attributes + zelf.set_attr("name", vm.unwrap_or_none(name), vm)?; + zelf.set_attr("obj", vm.unwrap_or_none(obj), vm)?; Ok(()) } @@ -1529,9 +1557,11 @@ pub(super) mod types { type Args = FuncArgs; fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // Only 'name', 'path', 'name_from' kwargs are allowed let mut kwargs = args.kwargs.clone(); let name = kwargs.swap_remove("name"); let path = kwargs.swap_remove("path"); + let name_from = kwargs.swap_remove("name_from"); // Check for any remaining invalid keyword arguments if let Some(invalid_key) = kwargs.keys().next() { @@ -1543,6 +1573,7 @@ pub(super) mod types { let dict = zelf.dict().unwrap(); dict.set_item("name", vm.unwrap_or_none(name), vm)?; dict.set_item("path", vm.unwrap_or_none(path), vm)?; + dict.set_item("name_from", vm.unwrap_or_none(name_from), vm)?; PyBaseException::slot_init(zelf, args, vm) } @@ -1592,11 +1623,45 @@ pub(super) mod types { #[repr(transparent)] pub struct PyMemoryError(PyException); - #[pyexception(name, base = PyException, ctx = "name_error", impl)] + #[pyexception(name, base = PyException, ctx = "name_error")] #[derive(Debug)] #[repr(transparent)] pub struct PyNameError(PyException); + // NameError_init: handles the .name. kwarg + #[pyexception(with(Initializer))] + impl PyNameError {} + + impl Initializer for PyNameError { + type Args = FuncArgs; + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // Only 'name' kwarg is allowed + let mut kwargs = args.kwargs.clone(); + let name = kwargs.swap_remove("name"); + + // Reject unknown kwargs + if let Some(invalid_key) = kwargs.keys().next() { + return Err(vm.new_type_error(format!( + "NameError() got an unexpected keyword argument '{invalid_key}'" + ))); + } + + // Pass args without kwargs to BaseException_init + let base_args = FuncArgs::new(args.args.clone(), KwArgs::default()); + PyBaseException::slot_init(zelf.clone(), base_args, vm)?; + + // Set name attribute if provided + if let Some(name) = name { + zelf.set_attr("name", name, vm)?; + } + Ok(()) + } + + fn init(_zelf: PyRef, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + #[pyexception(name, base = PyNameError, ctx = "unbound_local_error", impl)] #[derive(Debug)] #[repr(transparent)] @@ -2232,31 +2297,17 @@ pub(super) mod types { } } + // MiddlingExtendsException: inherits __init__ from SyntaxError via MRO #[pyexception( name = "_IncompleteInputError", base = PySyntaxError, - ctx = "incomplete_input_error" + ctx = "incomplete_input_error", + impl )] #[derive(Debug)] #[repr(transparent)] pub struct PyIncompleteInputError(PySyntaxError); - #[pyexception(with(Initializer))] - impl PyIncompleteInputError {} - - impl Initializer for PyIncompleteInputError { - type Args = FuncArgs; - - fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - zelf.set_attr("name", vm.ctx.new_str("SyntaxError"), vm)?; - PySyntaxError::slot_init(zelf, args, vm) - } - - fn init(_zelf: PyRef, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { - unreachable!("slot_init is defined") - } - } - #[pyexception(name, base = PySyntaxError, ctx = "indentation_error", impl)] #[derive(Debug)] #[repr(transparent)] diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 3e356466e5c..d83be841275 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -1168,6 +1168,29 @@ impl ExecutingFrame<'_> { }); Ok(None) } + Instruction::LoadFromDictOrGlobals(idx) => { + // PEP 649: Pop dict from stack (classdict), check there first, then globals + let dict = self.pop_value(); + let name = self.code.names[idx.get(arg) as usize]; + + // Only treat KeyError as "not found", propagate other exceptions + let value = if let Some(dict_obj) = dict.downcast_ref::() { + dict_obj.get_item_opt(name, vm)? + } else { + // Not an exact dict, use mapping protocol + match dict.get_item(name, vm) { + Ok(v) => Some(v), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => None, + Err(e) => return Err(e), + } + }; + + self.push_value(match value { + Some(v) => v, + None => self.load_global_or_builtin(name, vm)?, + }); + Ok(None) + } Instruction::LoadClosure(i) => { let value = self.cells_frees[i.get(arg) as usize].clone(); self.push_value(value.into()); diff --git a/crates/vm/src/stdlib/itertools.rs b/crates/vm/src/stdlib/itertools.rs index 1eedbde7a22..f1761315cee 100644 --- a/crates/vm/src/stdlib/itertools.rs +++ b/crates/vm/src/stdlib/itertools.rs @@ -2,23 +2,18 @@ pub(crate) use decl::make_module; #[pymodule(name = "itertools")] mod decl { - use crate::stdlib::itertools::decl::int::get_value; use crate::{ - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, PyWeakRef, TryFromObject, - VirtualMachine, - builtins::{ - PyGenericAlias, PyInt, PyIntRef, PyList, PyTuple, PyTupleRef, PyType, PyTypeRef, int, - tuple::IntoPyTuple, - }, + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, PyWeakRef, VirtualMachine, + builtins::{PyGenericAlias, PyInt, PyIntRef, PyList, PyTuple, PyType, PyTypeRef, int}, common::{ lock::{PyMutex, PyRwLock, PyRwLockWriteGuard}, rc::PyRc, }, convert::ToPyObject, - function::{ArgCallable, ArgIntoBool, FuncArgs, OptionalArg, OptionalOption, PosArgs}, + function::{ArgCallable, FuncArgs, OptionalArg, OptionalOption, PosArgs}, protocol::{PyIter, PyIterReturn, PyNumber}, raise_if_stop, - stdlib::{sys, warnings}, + stdlib::sys, types::{Constructor, IterNext, Iterable, Representable, SelfIter}, }; use crossbeam_utils::atomic::AtomicCell; @@ -28,15 +23,6 @@ mod decl { use alloc::fmt; use num_traits::{Signed, ToPrimitive}; - fn pickle_deprecation(vm: &VirtualMachine) -> PyResult<()> { - warnings::warn( - vm.ctx.exceptions.deprecation_warning, - "Itertool pickle/copy/deepcopy support will be removed in a Python 3.14.".to_owned(), - 1, - vm, - ) - } - #[pyattr] #[pyclass(name = "chain")] #[derive(Debug, PyPayload)] @@ -79,55 +65,6 @@ mod decl { ) -> PyGenericAlias { PyGenericAlias::from_args(cls, args, vm) } - - #[pymethod] - fn __reduce__(zelf: PyRef, vm: &VirtualMachine) -> PyResult { - pickle_deprecation(vm)?; - let source = zelf.source.read().clone(); - let active = zelf.active.read().clone(); - let cls = zelf.class().to_owned(); - let empty_tuple = vm.ctx.empty_tuple.clone(); - let reduced = match source { - Some(source) => match active { - Some(active) => vm.new_tuple((cls, empty_tuple, (source, active))), - None => vm.new_tuple((cls, empty_tuple, (source,))), - }, - None => vm.new_tuple((cls, empty_tuple)), - }; - Ok(reduced) - } - - #[pymethod] - fn __setstate__(zelf: PyRef, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { - let args = state.as_slice(); - if args.is_empty() { - return Err(vm.new_type_error("function takes at least 1 arguments (0 given)")); - } - if args.len() > 2 { - return Err(vm.new_type_error(format!( - "function takes at most 2 arguments ({} given)", - args.len() - ))); - } - let source = &args[0]; - if args.len() == 1 { - if !PyIter::check(source.as_ref()) { - return Err(vm.new_type_error("Arguments must be iterators.")); - } - *zelf.source.write() = source.to_owned().try_into_value(vm)?; - return Ok(()); - } - let active = &args[1]; - - if !PyIter::check(source.as_ref()) || !PyIter::check(active.as_ref()) { - return Err(vm.new_type_error("Arguments must be iterators.")); - } - let mut source_lock = zelf.source.write(); - let mut active_lock = zelf.active.write(); - *source_lock = source.to_owned().try_into_value(vm)?; - *active_lock = active.to_owned().try_into_value(vm)?; - Ok(()) - } } impl SelfIter for PyItertoolsChain {} @@ -209,16 +146,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsCompress { - #[pymethod] - fn __reduce__(zelf: PyRef, vm: &VirtualMachine) -> (PyTypeRef, (PyIter, PyIter)) { - let _ = pickle_deprecation(vm); - ( - zelf.class().to_owned(), - (zelf.data.clone(), zelf.selectors.clone()), - ) - } - } + impl PyItertoolsCompress {} impl SelfIter for PyItertoolsCompress {} @@ -275,16 +203,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor, Representable))] - impl PyItertoolsCount { - // TODO: Implement this - // if (lz->cnt == PY_SSIZE_T_MAX) - // return Py_BuildValue("0(00)", Py_TYPE(lz), lz->long_cnt, lz->long_step); - #[pymethod] - fn __reduce__(zelf: PyRef, vm: &VirtualMachine) -> (PyTypeRef, (PyObjectRef,)) { - let _ = pickle_deprecation(vm); - (zelf.class().to_owned(), (zelf.cur.read().clone(),)) - } - } + impl PyItertoolsCount {} impl SelfIter for PyItertoolsCount {} @@ -406,16 +325,6 @@ mod decl { .ok_or_else(|| vm.new_type_error("length of unsized object."))?; Ok(*times.read()) } - - #[pymethod] - fn __reduce__(zelf: PyRef, vm: &VirtualMachine) -> PyResult { - pickle_deprecation(vm)?; - let cls = zelf.class().to_owned(); - Ok(match zelf.times { - Some(ref times) => vm.new_tuple((cls, (zelf.object.clone(), *times.read()))), - None => vm.new_tuple((cls, (zelf.object.clone(),))), - }) - } } impl SelfIter for PyItertoolsRepeat {} @@ -474,19 +383,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsStarmap { - #[pymethod] - fn __reduce__( - zelf: PyRef, - vm: &VirtualMachine, - ) -> (PyTypeRef, (PyObjectRef, PyIter)) { - let _ = pickle_deprecation(vm); - ( - zelf.class().to_owned(), - (zelf.function.clone(), zelf.iterable.clone()), - ) - } - } + impl PyItertoolsStarmap {} impl SelfIter for PyItertoolsStarmap {} @@ -541,31 +438,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsTakewhile { - #[pymethod] - fn __reduce__( - zelf: PyRef, - vm: &VirtualMachine, - ) -> (PyTypeRef, (PyObjectRef, PyIter), u32) { - let _ = pickle_deprecation(vm); - ( - zelf.class().to_owned(), - (zelf.predicate.clone(), zelf.iterable.clone()), - zelf.stop_flag.load() as _, - ) - } - #[pymethod] - fn __setstate__( - zelf: PyRef, - state: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - if let Ok(obj) = ArgIntoBool::try_from_object(vm, state) { - zelf.stop_flag.store(obj.into_bool()); - } - Ok(()) - } - } + impl PyItertoolsTakewhile {} impl SelfIter for PyItertoolsTakewhile {} @@ -627,32 +500,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsDropwhile { - #[pymethod] - fn __reduce__( - zelf: PyRef, - vm: &VirtualMachine, - ) -> (PyTypeRef, (PyObjectRef, PyIter), u32) { - let _ = pickle_deprecation(vm); - ( - zelf.class().to_owned(), - (zelf.predicate.clone().into(), zelf.iterable.clone()), - (zelf.start_flag.load() as _), - ) - } - - #[pymethod] - fn __setstate__( - zelf: PyRef, - state: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - if let Ok(obj) = ArgIntoBool::try_from_object(vm, state) { - zelf.start_flag.store(obj.into_bool()); - } - Ok(()) - } - } + impl PyItertoolsDropwhile {} impl SelfIter for PyItertoolsDropwhile {} @@ -942,38 +790,6 @@ mod decl { .into_ref_with_type(vm, cls) .map(Into::into) } - - #[pymethod] - fn __reduce__(zelf: PyRef, vm: &VirtualMachine) -> PyResult { - pickle_deprecation(vm)?; - let cls = zelf.class().to_owned(); - let itr = zelf.iterable.clone(); - let cur = zelf.cur.take(); - let next = zelf.next.take(); - let step = zelf.step; - match zelf.stop { - Some(stop) => Ok(vm.new_tuple((cls, (itr, next, stop, step), (cur,)))), - _ => Ok(vm.new_tuple((cls, (itr, next, vm.new_pyobj(()), step), (cur,)))), - } - } - - #[pymethod] - fn __setstate__(zelf: PyRef, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { - let args = state.as_slice(); - if args.len() != 1 { - return Err(vm.new_type_error(format!( - "function takes exactly 1 argument ({} given)", - args.len() - ))); - } - let cur = &args[0]; - if let Ok(cur) = cur.try_to_value(vm) { - zelf.cur.store(cur); - } else { - return Err(vm.new_type_error("Argument must be usize.")); - } - Ok(()) - } } impl SelfIter for PyItertoolsIslice {} @@ -1037,19 +853,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsFilterFalse { - #[pymethod] - fn __reduce__( - zelf: PyRef, - vm: &VirtualMachine, - ) -> (PyTypeRef, (PyObjectRef, PyIter)) { - let _ = pickle_deprecation(vm); - ( - zelf.class().to_owned(), - (zelf.predicate.clone(), zelf.iterable.clone()), - ) - } - } + impl PyItertoolsFilterFalse {} impl SelfIter for PyItertoolsFilterFalse {} @@ -1106,59 +910,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsAccumulate { - #[pymethod] - fn __setstate__( - zelf: PyRef, - state: PyObjectRef, - _vm: &VirtualMachine, - ) -> PyResult<()> { - *zelf.acc_value.write() = Some(state); - Ok(()) - } - - #[pymethod] - fn __reduce__(zelf: PyRef, vm: &VirtualMachine) -> PyTupleRef { - let _ = pickle_deprecation(vm); - let class = zelf.class().to_owned(); - let bin_op = zelf.bin_op.clone(); - let it = zelf.iterable.clone(); - let acc_value = zelf.acc_value.read().clone(); - if let Some(initial) = &zelf.initial { - let chain_args = PyList::from(vec![initial.clone(), it.to_pyobject(vm)]); - let chain = PyItertoolsChain { - source: PyRwLock::new(Some(chain_args.to_pyobject(vm).get_iter(vm).unwrap())), - active: PyRwLock::new(None), - }; - let tup = vm.new_tuple((chain, bin_op)); - return vm.new_tuple((class, tup, acc_value)); - } - match acc_value { - Some(obj) if obj.is(&vm.ctx.none) => { - let chain_args = PyList::from(vec![]); - let chain = PyItertoolsChain { - source: PyRwLock::new(Some( - chain_args.to_pyobject(vm).get_iter(vm).unwrap(), - )), - active: PyRwLock::new(None), - } - .into_pyobject(vm); - let acc = Self { - iterable: PyIter::new(chain), - bin_op, - initial: None, - acc_value: PyRwLock::new(None), - }; - let tup = vm.new_tuple((acc, 1, None::)); - let islice_cls = PyItertoolsIslice::class(&vm.ctx).to_owned(); - return vm.new_tuple((islice_cls, tup)); - } - _ => {} - } - let tup = vm.new_tuple((it, bin_op)); - vm.new_tuple((class, tup, acc_value)) - } - } + impl PyItertoolsAccumulate {} impl SelfIter for PyItertoolsAccumulate {} @@ -1359,58 +1111,6 @@ mod decl { self.cur.store(idxs.len() - 1); } } - - #[pymethod] - fn __setstate__(zelf: PyRef, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { - let args = state.as_slice(); - if args.len() != zelf.pools.len() { - return Err(vm.new_type_error("Invalid number of arguments")); - } - let mut idxs: PyRwLockWriteGuard<'_, Vec> = zelf.idxs.write(); - idxs.clear(); - for s in 0..args.len() { - let index = get_value(state.get(s).unwrap()).to_usize().unwrap(); - let pool_size = zelf.pools.get(s).unwrap().len(); - if pool_size == 0 { - zelf.stop.store(true); - return Ok(()); - } - if index >= pool_size { - idxs.push(pool_size - 1); - } else { - idxs.push(index); - } - } - zelf.stop.store(false); - Ok(()) - } - - #[pymethod] - fn __reduce__(zelf: PyRef, vm: &VirtualMachine) -> PyTupleRef { - let _ = pickle_deprecation(vm); - let class = zelf.class().to_owned(); - - if zelf.stop.load() { - return vm.new_tuple((class, (vm.ctx.empty_tuple.clone(),))); - } - - let mut pools: Vec = Vec::new(); - for element in &zelf.pools { - pools.push(element.clone().into_pytuple(vm).into()); - } - - let mut indices: Vec = Vec::new(); - - for item in &zelf.idxs.read()[..] { - indices.push(vm.new_pyobj(*item)); - } - - vm.new_tuple(( - class, - pools.clone().into_pytuple(vm), - indices.into_pytuple(vm), - )) - } } impl SelfIter for PyItertoolsProduct {} @@ -1492,36 +1192,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsCombinations { - #[pymethod] - fn __reduce__(zelf: PyRef, vm: &VirtualMachine) -> PyTupleRef { - let _ = pickle_deprecation(vm); - let r = zelf.r.load(); - - let class = zelf.class().to_owned(); - - if zelf.exhausted.load() { - return vm.new_tuple(( - class, - vm.new_tuple((vm.ctx.empty_tuple.clone(), vm.ctx.new_int(r))), - )); - } - - let tup = vm.new_tuple((zelf.pool.clone().into_pytuple(vm), vm.ctx.new_int(r))); - - if zelf.result.read().is_none() { - vm.new_tuple((class, tup)) - } else { - let mut indices: Vec = Vec::new(); - - for item in &zelf.indices.read()[..r] { - indices.push(vm.new_pyobj(*item)); - } - - vm.new_tuple((class, tup, indices.into_pytuple(vm))) - } - } - } + impl PyItertoolsCombinations {} impl SelfIter for PyItertoolsCombinations {} impl IterNext for PyItertoolsCombinations { @@ -1730,16 +1401,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsPermutations { - #[pymethod] - fn __reduce__(zelf: PyRef, vm: &VirtualMachine) -> PyRef { - let _ = pickle_deprecation(vm); - vm.new_tuple(( - zelf.class().to_owned(), - vm.new_tuple((zelf.pool.clone(), vm.ctx.new_int(zelf.r.load()))), - )) - } - } + impl PyItertoolsPermutations {} impl SelfIter for PyItertoolsPermutations {} @@ -1846,32 +1508,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsZipLongest { - #[pymethod] - fn __reduce__(zelf: PyRef, vm: &VirtualMachine) -> PyResult { - pickle_deprecation(vm)?; - let args: Vec = zelf - .iterators - .iter() - .map(|i| i.clone().to_pyobject(vm)) - .collect(); - Ok(vm.new_tuple(( - zelf.class().to_owned(), - vm.new_tuple(args), - zelf.fillvalue.read().to_owned(), - ))) - } - - #[pymethod] - fn __setstate__( - zelf: PyRef, - state: PyObjectRef, - _vm: &VirtualMachine, - ) -> PyResult<()> { - *zelf.fillvalue.write() = state; - Ok(()) - } - } + impl PyItertoolsZipLongest {} impl SelfIter for PyItertoolsZipLongest {} diff --git a/crates/vm/src/stdlib/thread.rs b/crates/vm/src/stdlib/thread.rs index db588e5eab7..9f0c0535d71 100644 --- a/crates/vm/src/stdlib/thread.rs +++ b/crates/vm/src/stdlib/thread.rs @@ -1,8 +1,10 @@ //! Implementation of the _thread module +#[cfg(unix)] +pub(crate) use _thread::after_fork_child; #[cfg_attr(target_arch = "wasm32", allow(unused_imports))] pub(crate) use _thread::{ - CurrentFrameSlot, HandleEntry, RawRMutex, ShutdownEntry, after_fork_child, - get_all_current_frames, get_ident, init_main_thread_ident, make_module, + CurrentFrameSlot, HandleEntry, RawRMutex, ShutdownEntry, get_all_current_frames, get_ident, + init_main_thread_ident, make_module, }; #[pymodule] @@ -882,6 +884,7 @@ pub(crate) mod _thread { /// Called after fork() in child process to mark all other threads as done. /// This prevents join() from hanging on threads that don't exist in the child. + #[cfg(unix)] pub fn after_fork_child(vm: &VirtualMachine) { let current_ident = get_ident(); diff --git a/crates/vm/src/version.rs b/crates/vm/src/version.rs index 2b5e82a8d76..bd42341ef76 100644 --- a/crates/vm/src/version.rs +++ b/crates/vm/src/version.rs @@ -4,9 +4,9 @@ use chrono::{Local, prelude::DateTime}; use core::time::Duration; use std::time::UNIX_EPOCH; -// = 3.13.0alpha +// = 3.14.0alpha pub const MAJOR: usize = 3; -pub const MINOR: usize = 13; +pub const MINOR: usize = 14; pub const MICRO: usize = 0; pub const RELEASELEVEL: &str = "alpha"; pub const RELEASELEVEL_N: usize = 0xA; diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs index 65c742e4915..9978554bcc4 100644 --- a/crates/vm/src/vm/context.rs +++ b/crates/vm/src/vm/context.rs @@ -91,7 +91,10 @@ declare_const_name! { __all__, __and__, __anext__, + __annotate__, + __annotate_func__, __annotations__, + __annotations_cache__, __args__, __await__, __bases__, diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 8233df43a2b..27b70e321cf 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -523,7 +523,7 @@ impl VirtualMachine { /// ```no_run /// use rustpython_vm::Interpreter; /// Interpreter::without_stdlib(Default::default()).enter(|vm| { - /// let bytes = std::fs::read("__pycache__/.rustpython-313.pyc").unwrap(); + /// let bytes = std::fs::read("__pycache__/.rustpython-314.pyc").unwrap(); /// let main_scope = vm.new_scope_with_main().unwrap(); /// vm.run_pyc_bytes(&bytes, main_scope); /// }); diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 119444be75c..8fc8108b6a3 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -65,10 +65,6 @@ impl VirtualMachine { pub fn new_scope_with_main(&self) -> PyResult { let scope = self.new_scope_with_builtins(); let main_module = self.new_module("__main__", scope.globals.clone(), None); - main_module - .dict() - .set_item("__annotations__", self.ctx.new_dict().into(), self) - .expect("Failed to initialize __main__.__annotations__"); self.sys_module.get_attr("modules", self)?.set_item( "__main__", diff --git a/extra_tests/snippets/builtin_bool.py b/extra_tests/snippets/builtin_bool.py index 6b6b4e0e08b..902ed0cced0 100644 --- a/extra_tests/snippets/builtin_bool.py +++ b/extra_tests/snippets/builtin_bool.py @@ -18,7 +18,9 @@ assert bool(1) is True assert bool({}) is False -assert bool(NotImplemented) is True +# NotImplemented cannot be used in a boolean context (Python 3.14+) +with assert_raises(TypeError): + bool(NotImplemented) assert bool(...) is True if not 1: diff --git a/extra_tests/snippets/builtin_bytearray.py b/extra_tests/snippets/builtin_bytearray.py index 0b7e419390e..ee11e913ff2 100644 --- a/extra_tests/snippets/builtin_bytearray.py +++ b/extra_tests/snippets/builtin_bytearray.py @@ -153,16 +153,41 @@ class B(bytearray): # # hex from hex assert bytearray([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" -bytearray.fromhex("62 6c7a 34350a ") == b"blz45\n" +# fromhex with str +assert bytearray.fromhex("62 6c7a 34350a ") == b"blz45\n" + +# fromhex with bytes +assert bytearray.fromhex(b"62 6c7a 34350a ") == b"blz45\n" +assert bytearray.fromhex(b"B9 01EF") == b"\xb9\x01\xef" + +# fromhex with bytearray (bytes-like object) +assert bytearray.fromhex(bytearray(b"4142")) == b"AB" + +# fromhex with memoryview (bytes-like object) +assert bytearray.fromhex(memoryview(b"4142")) == b"AB" + +# fromhex error: non-hexadecimal character try: bytearray.fromhex("62 a 21") except ValueError as e: - str(e) == "non-hexadecimal number found in fromhex() arg at position 4" + assert str(e) == "non-hexadecimal number found in fromhex() arg at position 4" try: bytearray.fromhex("6Z2") except ValueError as e: - str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + assert str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + +# fromhex error: odd number of hex digits +try: + bytearray.fromhex("abc") +except ValueError as e: + assert str(e) == "fromhex() arg must contain an even number of hexadecimal digits" + +# fromhex error: wrong type with assert_raises(TypeError): + bytearray.fromhex(123) + +# fromhex with bytes containing invalid hex raises ValueError +with assert_raises(ValueError): bytearray.fromhex(b"hhjjk") # center assert [bytearray(b"koki").center(i, b"|") for i in range(3, 10)] == [ diff --git a/extra_tests/snippets/builtin_bytes.py b/extra_tests/snippets/builtin_bytes.py index 4cb743baa6f..2cb4c317f49 100644 --- a/extra_tests/snippets/builtin_bytes.py +++ b/extra_tests/snippets/builtin_bytes.py @@ -137,16 +137,41 @@ # hex from hex assert bytes([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" -bytes.fromhex("62 6c7a 34350a ") == b"blz45\n" +# fromhex with str +assert bytes.fromhex("62 6c7a 34350a ") == b"blz45\n" + +# fromhex with bytes +assert bytes.fromhex(b"62 6c7a 34350a ") == b"blz45\n" +assert bytes.fromhex(b"B9 01EF") == b"\xb9\x01\xef" + +# fromhex with bytearray (bytes-like object) +assert bytes.fromhex(bytearray(b"4142")) == b"AB" + +# fromhex with memoryview (bytes-like object) +assert bytes.fromhex(memoryview(b"4142")) == b"AB" + +# fromhex error: non-hexadecimal character try: bytes.fromhex("62 a 21") except ValueError as e: - str(e) == "non-hexadecimal number found in fromhex() arg at position 4" + assert str(e) == "non-hexadecimal number found in fromhex() arg at position 4" try: bytes.fromhex("6Z2") except ValueError as e: - str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + assert str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + +# fromhex error: odd number of hex digits +try: + bytes.fromhex("abc") +except ValueError as e: + assert str(e) == "fromhex() arg must contain an even number of hexadecimal digits" + +# fromhex error: wrong type with assert_raises(TypeError): + bytes.fromhex(123) + +# fromhex with bytes containing invalid hex raises ValueError +with assert_raises(ValueError): bytes.fromhex(b"hhjjk") # center assert [b"koki".center(i, b"|") for i in range(3, 10)] == [ diff --git a/extra_tests/snippets/builtin_exceptions.py b/extra_tests/snippets/builtin_exceptions.py index 246be3b8fda..8879e130bc2 100644 --- a/extra_tests/snippets/builtin_exceptions.py +++ b/extra_tests/snippets/builtin_exceptions.py @@ -239,12 +239,10 @@ class SubError(MyError): raise e except MyError as exc: # It was a segmentation fault before, will print info to stdout: - if platform.python_implementation() == "RustPython": - # For some reason `CPython` hangs on this code: - sys.excepthook(type(exc), exc, exc.__traceback__) - assert isinstance(exc, MyError) - assert exc.__cause__ is None - assert exc.__context__ is e + sys.excepthook(type(exc), exc, exc.__traceback__) + assert isinstance(exc, MyError) + assert exc.__cause__ is None + assert exc.__context__ is e # Regression to @@ -255,26 +253,42 @@ class SubError(MyError): assert BaseException.__init__.__qualname__ == "BaseException.__init__" assert BaseException().__dict__ == {} +# Exception inherits __init__ from BaseException assert Exception.__new__.__qualname__ == "Exception.__new__", ( Exception.__new__.__qualname__ ) -assert Exception.__init__.__qualname__ == "Exception.__init__", ( +assert Exception.__init__.__qualname__ == "BaseException.__init__", ( Exception.__init__.__qualname__ ) assert Exception().__dict__ == {} -# Extends `BaseException`, simple: +# Extends `BaseException`, simple - inherits __init__ from BaseException: assert KeyboardInterrupt.__new__.__qualname__ == "KeyboardInterrupt.__new__", ( KeyboardInterrupt.__new__.__qualname__ ) -assert KeyboardInterrupt.__init__.__qualname__ == "KeyboardInterrupt.__init__" +assert KeyboardInterrupt.__init__.__qualname__ == "BaseException.__init__" assert KeyboardInterrupt().__dict__ == {} -# Extends `Exception`, simple: +# Extends `BaseException`, complex - has its own __init__: +# SystemExit_init sets self.code based on args length +assert SystemExit.__init__.__qualname__ == "SystemExit.__init__" +assert SystemExit.__dict__.get("__init__") is not None, ( + "SystemExit must have its own __init__" +) +assert SystemExit.__init__ is not BaseException.__init__ +assert SystemExit().__dict__ == {} +# SystemExit.code behavior: +assert SystemExit().code is None +assert SystemExit(1).code == 1 +assert SystemExit(1, 2).code == (1, 2) +assert SystemExit(1, 2, 3).code == (1, 2, 3) + + +# Extends `Exception`, simple - inherits __init__ from BaseException: assert TypeError.__new__.__qualname__ == "TypeError.__new__" -assert TypeError.__init__.__qualname__ == "TypeError.__init__" +assert TypeError.__init__.__qualname__ == "BaseException.__init__" assert TypeError().__dict__ == {} @@ -356,7 +370,8 @@ class SubError(MyError): # Custom `__new__` and `__init__`: assert ImportError.__init__.__qualname__ == "ImportError.__init__" assert ImportError(name="a").name == "a" -assert ModuleNotFoundError.__init__.__qualname__ == "ModuleNotFoundError.__init__" +# ModuleNotFoundError inherits __init__ from ImportError via MRO (MiddlingExtendsException) +assert ModuleNotFoundError.__init__.__qualname__ == "ImportError.__init__" assert ModuleNotFoundError(name="a").name == "a" diff --git a/extra_tests/snippets/builtin_int.py b/extra_tests/snippets/builtin_int.py index bc3cd5fd996..aab24cbb4cc 100644 --- a/extra_tests/snippets/builtin_int.py +++ b/extra_tests/snippets/builtin_int.py @@ -318,8 +318,9 @@ def __int__(self): assert isinstance((1).__round__(0), int) assert (0).__round__(0) == 0 assert (1).__round__(0) == 1 -assert_raises(TypeError, lambda: (0).__round__(None)) -assert_raises(TypeError, lambda: (1).__round__(None)) +# Python 3.14+: __round__(None) is now allowed, same as __round__() +assert (0).__round__(None) == 0 +assert (1).__round__(None) == 1 assert_raises(TypeError, lambda: (0).__round__(0.0)) assert_raises(TypeError, lambda: (1).__round__(0.0)) diff --git a/extra_tests/snippets/code_co_consts.py b/extra_tests/snippets/code_co_consts.py index 58355652682..13f76a0d13e 100644 --- a/extra_tests/snippets/code_co_consts.py +++ b/extra_tests/snippets/code_co_consts.py @@ -1,39 +1,112 @@ +""" +Test co_consts behavior for Python 3.14+ + +In Python 3.14+: +- Functions with docstrings have the docstring as co_consts[0] +- CO_HAS_DOCSTRING flag (0x4000000) indicates docstring presence +- Functions without docstrings do NOT have None added as placeholder for docstring + +Note: Other constants (small integers, code objects, etc.) may still appear in co_consts +depending on optimization level. This test focuses on docstring behavior. +""" + + +# Test function with docstring - docstring should be co_consts[0] +def with_doc(): + """This is a docstring""" + return 1 + + +assert with_doc.__code__.co_consts[0] == "This is a docstring", ( + with_doc.__code__.co_consts +) +assert with_doc.__doc__ == "This is a docstring" +# Check CO_HAS_DOCSTRING flag (0x4000000) +assert with_doc.__code__.co_flags & 0x4000000, hex(with_doc.__code__.co_flags) + + +# Test function without docstring - should NOT have HAS_DOCSTRING flag +def no_doc(): + return 1 + + +assert not (no_doc.__code__.co_flags & 0x4000000), hex(no_doc.__code__.co_flags) +assert no_doc.__doc__ is None + + +# Test async function with docstring from asyncio import sleep -def f(): - def g(): - return 1 +async def async_with_doc(): + """Async docstring""" + await sleep(1) + return 1 - assert g.__code__.co_consts[0] == None - return 2 +assert async_with_doc.__code__.co_consts[0] == "Async docstring", ( + async_with_doc.__code__.co_consts +) +assert async_with_doc.__doc__ == "Async docstring" +assert async_with_doc.__code__.co_flags & 0x4000000 -assert f.__code__.co_consts[0] == None +# Test async function without docstring +async def async_no_doc(): + await sleep(1) + return 1 + + +assert not (async_no_doc.__code__.co_flags & 0x4000000) +assert async_no_doc.__doc__ is None -def generator(): + +# Test generator with docstring +def gen_with_doc(): + """Generator docstring""" yield 1 yield 2 -assert generator().gi_code.co_consts[0] == None +assert gen_with_doc.__code__.co_consts[0] == "Generator docstring" +assert gen_with_doc.__doc__ == "Generator docstring" +assert gen_with_doc.__code__.co_flags & 0x4000000 -async def async_f(): - await sleep(1) - return 1 +# Test generator without docstring +def gen_no_doc(): + yield 1 + yield 2 + +assert not (gen_no_doc.__code__.co_flags & 0x4000000) +assert gen_no_doc.__doc__ is None -assert async_f.__code__.co_consts[0] == None +# Test lambda - cannot have docstring lambda_f = lambda: 0 -assert lambda_f.__code__.co_consts[0] == None +assert not (lambda_f.__code__.co_flags & 0x4000000) +assert lambda_f.__doc__ is None + + +# Test class method with docstring +class cls_with_doc: + def method(): + """Method docstring""" + return 1 + +assert cls_with_doc.method.__code__.co_consts[0] == "Method docstring" +assert cls_with_doc.method.__doc__ == "Method docstring" -class cls: - def f(): + +# Test class method without docstring +class cls_no_doc: + def method(): return 1 -assert cls().f.__code__.co_consts[0] == None +assert not (cls_no_doc.method.__code__.co_flags & 0x4000000) +assert cls_no_doc.method.__doc__ is None + +print("All co_consts tests passed!") diff --git a/extra_tests/snippets/example_interactive.py b/extra_tests/snippets/example_interactive.py index f9484f15dcf..5958dd11707 100644 --- a/extra_tests/snippets/example_interactive.py +++ b/extra_tests/snippets/example_interactive.py @@ -4,7 +4,7 @@ def f(x, y, *args, power=1, **kwargs): - print("Constant String", 2, None, (2, 4)) + print("Constant String", 256, None, (2, 4)) assert code_class == type(c1) z = x * y return z**power @@ -19,7 +19,7 @@ def f(x, y, *args, power=1, **kwargs): # assert isinstance(c2.co_code, bytes) assert "Constant String" in c2.co_consts, c2.co_consts print(c2.co_consts) -assert 2 in c2.co_consts, c2.co_consts +assert 256 in c2.co_consts, c2.co_consts assert "example_interactive.py" in c2.co_filename assert c2.co_firstlineno == 6, str(c2.co_firstlineno) # assert isinstance(c2.co_flags, int) # 'OPTIMIZED, NEWLOCALS, NOFREE' diff --git a/extra_tests/snippets/stdlib_itertools.py b/extra_tests/snippets/stdlib_itertools.py index 4d2e9f6e1f7..ce7a494713a 100644 --- a/extra_tests/snippets/stdlib_itertools.py +++ b/extra_tests/snippets/stdlib_itertools.py @@ -1,5 +1,4 @@ import itertools -import pickle from testutils import assert_raises @@ -181,10 +180,6 @@ # itertools.takewhile tests -def underten(x): - return x < 10 - - from itertools import takewhile as tw t = tw(lambda n: n < 5, [1, 2, 5, 1, 3]) @@ -226,70 +221,6 @@ def underten(x): with assert_raises(StopIteration): next(t) -it = tw(underten, [1, 3, 5, 20, 2, 4, 6, 8]) -assert ( - pickle.dumps(it, 0) - == b"citertools\ntakewhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI0\nbtp6\nRp7\nI0\nb." -) -assert ( - pickle.dumps(it, 1) - == b"citertools\ntakewhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x00btq\x06Rq\x07K\x00b." -) -assert ( - pickle.dumps(it, 2) - == b"\x80\x02citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b." -) -assert ( - pickle.dumps(it, 3) - == b"\x80\x03citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b." -) -assert ( - pickle.dumps(it, 4) - == b"\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b." -) -assert ( - pickle.dumps(it, 5) - == b"\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b." -) -next(it) -next(it) -next(it) -try: - next(it) -except StopIteration: - pass -assert ( - pickle.dumps(it, 0) - == b"citertools\ntakewhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI4\nbtp6\nRp7\nI1\nb." -) -assert ( - pickle.dumps(it, 1) - == b"citertools\ntakewhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x04btq\x06Rq\x07K\x01b." -) -assert ( - pickle.dumps(it, 2) - == b"\x80\x02citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b." -) -assert ( - pickle.dumps(it, 3) - == b"\x80\x03citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b." -) -assert ( - pickle.dumps(it, 4) - == b"\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b." -) -assert ( - pickle.dumps(it, 5) - == b"\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b." -) -for proto in range(pickle.HIGHEST_PROTOCOL + 1): - try: - next(pickle.loads(pickle.dumps(it, proto))) - assert False - except StopIteration: - pass - - # itertools.islice tests @@ -297,40 +228,28 @@ def assert_matches_seq(it, seq): assert list(it) == list(seq) -def test_islice_pickle(it): - for p in range(pickle.HIGHEST_PROTOCOL + 1): - it == pickle.loads(pickle.dumps(it, p)) - - i = itertools.islice it = i([1, 2, 3, 4, 5], 3) assert_matches_seq(it, [1, 2, 3]) -test_islice_pickle(it) it = i([0.5, 1, 1.5, 2, 2.5, 3, 4, 5], 1, 6, 2) assert_matches_seq(it, [1, 2, 3]) -test_islice_pickle(it) it = i([1, 2], None) assert_matches_seq(it, [1, 2]) -test_islice_pickle(it) it = i([1, 2, 3], None, None, None) assert_matches_seq(it, [1, 2, 3]) -test_islice_pickle(it) it = i([1, 2, 3], 1, None, None) assert_matches_seq(it, [2, 3]) -test_islice_pickle(it) it = i([1, 2, 3], None, 2, None) assert_matches_seq(it, [1, 2]) -test_islice_pickle(it) it = i([1, 2, 3], None, None, 3) assert_matches_seq(it, [1]) -test_islice_pickle(it) # itertools.filterfalse it = itertools.filterfalse(lambda x: x % 2, range(10)) @@ -359,59 +278,6 @@ def test_islice_pickle(it): with assert_raises(StopIteration): next(it) -it = itertools.dropwhile(underten, [1, 3, 5, 20, 2, 4, 6, 8]) -assert ( - pickle.dumps(it, 0) - == b"citertools\ndropwhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI0\nbtp6\nRp7\nI0\nb." -) -assert ( - pickle.dumps(it, 1) - == b"citertools\ndropwhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x00btq\x06Rq\x07K\x00b." -) -assert ( - pickle.dumps(it, 2) - == b"\x80\x02citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b." -) -assert ( - pickle.dumps(it, 3) - == b"\x80\x03citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b." -) -assert ( - pickle.dumps(it, 4) - == b"\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b." -) -assert ( - pickle.dumps(it, 5) - == b"\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b." -) -next(it) -assert ( - pickle.dumps(it, 0) - == b"citertools\ndropwhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI4\nbtp6\nRp7\nI1\nb." -) -assert ( - pickle.dumps(it, 1) - == b"citertools\ndropwhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x04btq\x06Rq\x07K\x01b." -) -assert ( - pickle.dumps(it, 2) - == b"\x80\x02citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b." -) -assert ( - pickle.dumps(it, 3) - == b"\x80\x03citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b." -) -assert ( - pickle.dumps(it, 4) - == b"\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b." -) -assert ( - pickle.dumps(it, 5) - == b"\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b." -) -for proto in range(pickle.HIGHEST_PROTOCOL + 1): - assert next(pickle.loads(pickle.dumps(it, proto))) == 2 - # itertools.accumulate it = itertools.accumulate([6, 3, 7, 1, 0, 9, 8, 8]) diff --git a/extra_tests/snippets/stdlib_socket.py b/extra_tests/snippets/stdlib_socket.py index b49fdcf08c2..3f56d2b926e 100644 --- a/extra_tests/snippets/stdlib_socket.py +++ b/extra_tests/snippets/stdlib_socket.py @@ -131,8 +131,9 @@ with assert_raises(OSError): socket.inet_aton("test") -with assert_raises(OverflowError): - socket.htonl(-1) +# TODO: RUSTPYTHON +# with assert_raises(ValueError): +# socket.htonl(-1) assert socket.htonl(0) == 0 assert socket.htonl(10) == 167772160 diff --git a/extra_tests/snippets/stdlib_typing.py b/extra_tests/snippets/stdlib_typing.py index 681790abd0b..07348945842 100644 --- a/extra_tests/snippets/stdlib_typing.py +++ b/extra_tests/snippets/stdlib_typing.py @@ -12,3 +12,26 @@ def abort_signal_handler( # Ensure PEP 604 unions work with typing.Callable aliases. TracebackFilter = bool | Callable[[int], int] + + +# Test that Union/Optional in function parameter annotations work correctly. +# This tests that annotation scopes can access global implicit symbols (like Union) +# that are imported at module level but not explicitly bound in the function scope. +# Regression test for: rich +from typing import Optional, Union + + +def function_with_union_param(x: Optional[Union[int, str]] = None) -> None: + pass + + +class ClassWithUnionParams: + def __init__( + self, + color: Optional[Union[str, int]] = None, + bold: Optional[bool] = None, + ) -> None: + pass + + def method(self, value: Union[int, float]) -> Union[str, bytes]: + return str(value) diff --git a/extra_tests/snippets/syntax_assignment.py b/extra_tests/snippets/syntax_assignment.py index 8635dc5d795..851558a9db0 100644 --- a/extra_tests/snippets/syntax_assignment.py +++ b/extra_tests/snippets/syntax_assignment.py @@ -59,7 +59,18 @@ def g(): assert a == 1337 assert b == False -assert __annotations__['a'] == bool +# PEP 649: In Python 3.14, __annotations__ is not automatically defined at module level +# Accessing it raises NameError +from testutils import assert_raises + +with assert_raises(NameError): + __annotations__ + +# Use __annotate__ to get annotations (PEP 649) +assert callable(__annotate__) +annotations = __annotate__(1) # 1 = FORMAT_VALUE +assert annotations['a'] == bool +assert annotations['b'] == bool n = 0 diff --git a/extra_tests/snippets/syntax_function2.py b/extra_tests/snippets/syntax_function2.py index d0901af6a14..4a04acd51c1 100644 --- a/extra_tests/snippets/syntax_function2.py +++ b/extra_tests/snippets/syntax_function2.py @@ -80,6 +80,7 @@ def nested(): def f7(): + # PEP 649: annotations are deferred, so void is not evaluated at definition time try: def t() -> void: # noqa: F821 pass @@ -87,7 +88,7 @@ def t() -> void: # noqa: F821 return True return False -assert f7() +assert not f7() # PEP 649: no NameError because annotation is deferred def f8() -> int: diff --git a/scripts/auto_mark_test.py b/scripts/auto_mark_test.py index 78230954162..2e3aef52599 100644 --- a/scripts/auto_mark_test.py +++ b/scripts/auto_mark_test.py @@ -24,6 +24,7 @@ import argparse import ast +import re import shutil import sys from pathlib import Path @@ -82,7 +83,7 @@ def parse_results(result): test_results.stdout = result.stdout in_test_results = False for line in lines: - if line == "Run tests sequentially": + if re.match(r"Run tests? sequentially", line): in_test_results = True elif line.startswith("-----------"): in_test_results = False diff --git a/whats_left.py b/whats_left.py index c5b0be6eadc..99b840f9da7 100755 --- a/whats_left.py +++ b/whats_left.py @@ -1,6 +1,6 @@ #!/usr/bin/env -S python3 -I # /// script -# requires-python = ">=3.13" +# requires-python = ">=3.14" # /// # This script generates Lib/snippets/whats_left_data.py with these variables defined: @@ -37,9 +37,9 @@ implementation = platform.python_implementation() if implementation != "CPython": sys.exit(f"whats_left.py must be run under CPython, got {implementation} instead") -if sys.version_info[:2] < (3, 13): +if sys.version_info[:2] < (3, 14): sys.exit( - f"whats_left.py must be run under CPython 3.13 or newer, got {implementation} {sys.version} instead. If you have uv, try `uv run python -I whats_left.py` to select a proper Python interpreter easier." + f"whats_left.py must be run under CPython 3.14 or newer, got {implementation} {sys.version} instead. If you have uv, try `uv run python -I whats_left.py` to select a proper Python interpreter easier." ) @@ -195,6 +195,9 @@ def gen_methods(): typ = eval(typ_code) attrs = [] for attr in dir(typ): + # Skip attributes in dir() but not actually accessible (e.g., descriptor that raises) + if not hasattr(typ, attr): + continue if attr_is_not_inherited(typ, attr): attrs.append((attr, extra_info(getattr(typ, attr)))) methods[typ.__name__] = (typ_code, extra_info(typ), attrs)